package org.pathways.openciss.server;

import java.io.BufferedInputStream;
//import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
//import java.io.InputStreamReader;
//import java.io.OutputStream;
//import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.math.BigInteger;
//import java.io.Reader;
import java.nio.channels.Channels;
//import java.nio.channels.ReadableByteChannel;
//import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.zip.GZIPInputStream;
//import java.util.zip.ZipEntry;
//import java.util.zip.ZipInputStream;

import javax.persistence.EntityManager;
import javax.persistence.NoResultException;
import javax.persistence.Query;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
//Stax
//import javax.xml.stream.XMLInputFactory;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
//import javax.xml.bind.JAXBException;
//import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
//import javax.xml.datatype.XMLGregorianCalendar;
//import javax.xml.namespace.QName;
//import javax.xml.parsers.DocumentBuilder;
//import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamReader;
//import javax.xml.transform.OutputKeys;
//import javax.xml.transform.Transformer;
//import javax.xml.transform.TransformerException;
//import javax.xml.transform.TransformerFactory;
//import javax.xml.transform.dom.DOMSource;
//import javax.xml.transform.stream.StreamResult;
//import javax.xml.transform.stream.StreamSource;
//import javax.xml.xpath.XPath;
//import javax.xml.xpath.XPathConstants;
//import javax.xml.xpath.XPathExpressionException;
//import javax.xml.xpath.XPathFactory;
import com.google.appengine.api.blobstore.BlobKey;
import com.google.appengine.api.blobstore.BlobstoreInputStream;
//import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.files.AppEngineFile;
//import com.google.appengine.api.files.FileReadChannel;
import com.google.appengine.api.files.FileService;
import com.google.appengine.api.files.FileServiceFactory;
import com.google.appengine.api.files.FileWriteChannel;

import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
import org.pathways.openciss.info.hmis.schema._3_0.airs_3_0_mod.TSite;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Agency;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.HealthStatus;
//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.FundingSourcesServiceEvent;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.FundingSourceServiceEvent;
//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Export;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Household;
//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.IDIndex;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.IncomeAndSource;
//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.IncomeAndSources;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.IncomeSourceCode;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Member;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Members;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.NonNegativeInteger;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Person;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.PersonAddress;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.PersonHistorical;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.PhoneNumber;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.PriorResidence;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.SevenValDKRefused2;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.SixValDKRefused2;
//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Service;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.ServiceEvent;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.SiteService;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.SiteServiceParticipation;
import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Source;
import org.pathways.openciss.model.RosieIndexes;
import org.pathways.openciss.shared.EMF;
import org.pathways.openciss.shared.HUDXML3Blob;
import org.pathways.openciss.shared.HUDXML3BlobService;
import org.pathways.openciss.shared.RosieIndexesService;

//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Sources;
//import org.pathways.openciss.info.hmis.schema._3_0.hud_hmis.Sources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
//import org.w3c.dom.Document;
//import org.w3c.dom.Element;
//import org.w3c.dom.NamedNodeMap;
//import org.w3c.dom.NodeList;
//import org.w3c.dom.Node;
import au.com.bytecode.opencsv.CSVWriter;


public class RosieCSVBlobServlet extends HttpServlet {
    
	private static final transient Logger log = LoggerFactory.getLogger(RosieCSVBlobServlet.class);

	private static final long serialVersionUID = 1L;
	AppEngineFile fileRosieCSV1FamilyInformation;
	AppEngineFile fileRosieCSV2FamilyMembers;
	AppEngineFile fileRosieCSV3FamilyIncomeAndExpense;
	AppEngineFile fileRosieCSV4MemberHealthProblems;
	AppEngineFile fileRosieCSV6FamilyAssistanceProvided;
	List <Agency> agencyList;
	List <Household> householdList;
	List<Person> personList;
	List <Source> sourceList;
	//DocumentBuilderFactory domFactory;
	//XPath xpath;
	//DocumentBuilder domBuilder;
	//Document doc;
	//XPathFactory factory;
	Integer siteServiceID;
	FileService fileService;
	JAXBElement<?> sourcesJAXBElement;
	AppEngineFile fileHUDXML;
	XMLInputFactory xmlif;
	//XMLStreamReader xmlr;
	Unmarshaller unmarshaller;

	
	static String[] file1Record = {"AgencyCode","TransactionDate_mmddyy","TransactionTime_hhmmss",
		"SSNFamilyHead", "RecordType_1","StreetAddress","Apartment","City","State",
		"Zip5","StreetAddress","Apartment","City","State","Zip5","WorkTelephone",
		"HomeTelephone","ReasonForCrisis_Code","FamilySize","HousingSituation_Code",
		"LengthInHousingSituation_Code","NumberNeedingShelter","InsuranceType1_Code",
		"InsuranceType2_Code","ChronicHomeless_Y_N","Prior_Loc_Y_N"};
	static String[] file2Record = {"AgencyCode","TransactionDate_mmddyy","TransactionTime_hhmmss",
		"SSNFamilyHead","RecordType_2","FamilyMemberNumber","FamilyMemberSSN",
		"Relationship_Code","FirstName","MiddleName","LastName","DateOfBirth_mmddyy",
		"Gender_M_F","Race1_Code","Race2_Code","Hispanic_Y_N","Pregnant_Y_N",
		"MonthsPregnant","NeedShelter_Y_N","Veteran_Code","StreetAddress","Apartment",
		"City","State","Zip5","HousingSituation_Code","LengthInHousingSituation_Code"};
	static String[] file3Record = {"AgencyCode","TransactionDate_mmddyy","TransactionTime_hhmmss",
		"SSNFamilyHead","RecordType_3","FamilyMemberNumber","Income_ExpenseType_Code",
	"MonthlyAmount"};
	static String[] file4Record = {"AgencyCode","TransactionDate_mmddyy","TransactionTime_hhmmss",
		"SSNFamilyHead","RecordType_4","FamilyMemberNumber","HealthProblem_Code",
	"HealthProblemDescription"};
	static String[] file6Record = {"AgencyCode","TransactionDate_mmddyy","TransactionTime_hhmmss",
		"SSNFamilyHead","RecordType_6","FamilyMemberNumber","AssistanceProvided_Code",
		"AssistanceProvided_Description","AssistanceProvided_Amount"};

	public void init() {
		try {
			fileService = FileServiceFactory.getFileService();
			xmlif = XMLInputFactory.newInstance();
		} catch (Exception e){}
	}

	@Override
	public void doGet(HttpServletRequest req, HttpServletResponse resp)
		throws ServletException, IOException {
		
		log.info("doGet in RosieCSVBlobServlet entered");
		BlobKey hudXMLURL = new BlobKey(req.getParameter("hud_xml_url"));
		String[] blobStoreURL = new String[5];
		// check the completion state of each of the files by reading from the datastore
		    // look at the last/most recent record (either in file or in datastore/cloud sql)
		
		// if file1 already started and not finished, resume file 1
			// get last marker processed from file or db?  what is the key?
			// continue, writing out each time
		HUDXML3BlobService hxbs = new HUDXML3BlobService();
		log.info("trying to get HUDXML3Blob object from Key String: " + hudXMLURL.getKeyString());
		HUDXML3Blob blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
		if (blob != null) {
			String id = blob.getId();
			if (!recordExists(id)) {
				log.info("for HUD XML blob record: " + id + " nothing has been done, not even file 1 indexing, so start up first time.");
				blobStoreURL = startUpFirstTime(id, hudXMLURL);
			} 
			else {
				log.info("HUD XML blob record: " + id + " already exists, so continuing from past session.");
				//need to make the fileWriters
				makeAllFileWritersFromPreexistingFiles(blob, hudXMLURL);
				
				if (!finishedConverting(id,1)) {
					convertHUDXMLToRosieCSV1(id, hudXMLURL);
					// set as finished in database
					blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
					blob.setCsvUrl1Completed(1);
					hxbs.updateHUDXML3Blob(blob);
				}
				// if file2 already started and not finished, resume file 2
				if (!finishedConverting(id, 2)) {
					convertHUDXMLToRosieCSV2(id, hudXMLURL);
					// set as finished in database
					blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
					blob.setCsvUrl2Completed(1);
					hxbs.updateHUDXML3Blob(blob);
				}

				//close out all files
				blobStoreURL = closeOutAllFiles();
			}
		}
		else {
			log.info("We have a problem, since the database didn't create an entry when uploading the HUDXML file");
		}
		//if no record for such a url, do all the things we always do to start things up as before

		//make response (this isn't really being used right now by the client, because of the long/unknown processing time and shutdowns
		resp.setHeader("Content-Type", "application/json");
		//create JSON response of blobStoreURL contents
		ObjectMapper mapper = new ObjectMapper();
		ArrayNode blobStoreURLs = mapper.createArrayNode();
		for (int i=0; i<blobStoreURL.length; i++) {
			blobStoreURLs.add(blobStoreURL[i]);  
		}
		log.info("Responding with blobStoreURLs.toString()" + blobStoreURLs.toString());
		resp.getOutputStream().println(blobStoreURLs.toString());	
	}
	
	String[] startUpFirstTime(String id, BlobKey hudXMLURL) {
		log.info("inside startUpFirstTime");
		//clear database of RosieIndexes
		RosieIndexesService ris = new RosieIndexesService();
		log.info("calling delete all Rosie indexes");
		ris.deleteAllRosieIndexes();
		String[] csvUrls;
		//blobKeyHUDXML =  hudXMLURL;

		makeHeaders(hudXMLURL);
		indexCSV1(id, hudXMLURL);
		indexCSV2(id, hudXMLURL);
		convertHUDXMLToRosieCSV1(id, hudXMLURL);
		// set as finished in database
		HUDXML3BlobService hxbs = new HUDXML3BlobService();
		HUDXML3Blob blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
		blob.setCsvUrl1Completed(1);
		hxbs.updateHUDXML3Blob(blob);
		blob = null;
		convertHUDXMLToRosieCSV2(id, hudXMLURL);
		blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
		blob.setCsvUrl2Completed(1);
		hxbs.updateHUDXML3Blob(blob);
		blob = null;
		csvUrls = closeOutAllFiles();
		log.info("finished processing in one pass!");
		return csvUrls;
	}
	
	String[] closeOutAllFiles() {
	// close everything out.  First close the existing writer which doesn't have a lock, then acquire a writer which does have a lock and close finally (no more writing)
		
		AppEngineFile csvFile = null;
		//String path; //url id path to CSV file in blobstorage to return 
		String[] csvUrls = new String[5];

		for (int i=1; i<6; i++) {  
			switch (i) {
				case 1: csvFile = fileRosieCSV1FamilyInformation;
				log.info("closing out file 1");
				try {
					//locked set to true
					FileWriteChannel fwc1 = fileService.openWriteChannel(csvFile, true);
					fwc1.closeFinally();
				} catch (IOException ioe) {ioe.printStackTrace();}
				// You can't get the blob key until the file is finalized
				BlobKey blobKeyRosieCSV1 = fileService.getBlobKey(csvFile);
				log.info("csv blob storage key is:" + blobKeyRosieCSV1.getKeyString());
				csvUrls[i-1] = blobKeyRosieCSV1.getKeyString();
				break;
				
				case 2: csvFile = fileRosieCSV2FamilyMembers;
				log.info("closing out file 2");
				try {
					//locked set to true
					FileWriteChannel fwc2 = fileService.openWriteChannel(csvFile, true);
					fwc2.closeFinally();
				} catch (IOException ioe) {ioe.printStackTrace();}
				// You can't get the blob key until the file is finalized
				BlobKey blobKeyRosieCSV2 = fileService.getBlobKey(csvFile);
				log.info("csv blob storage key is:" + blobKeyRosieCSV2.getKeyString());
				csvUrls[i-1] = blobKeyRosieCSV2.getKeyString();
				break;

				case 3: csvFile = fileRosieCSV3FamilyIncomeAndExpense;
				log.info("closing out file 3");
				try {
					//locked set to true
					FileWriteChannel fwc3 = fileService.openWriteChannel(csvFile, true);
					fwc3.closeFinally();
				} catch (IOException ioe) {ioe.printStackTrace();}
				// You can't get the blob key until the file is finalized
				BlobKey blobKeyRosieCSV3 = fileService.getBlobKey(csvFile);
				log.info("csv blob storage key is:" + blobKeyRosieCSV3.getKeyString());
				csvUrls[i-1] = blobKeyRosieCSV3.getKeyString();
				break;

				case 4: csvFile = fileRosieCSV4MemberHealthProblems;
				log.info("closing out file 4");
				try {
					//locked set to true
					FileWriteChannel fwc4 = fileService.openWriteChannel(csvFile, true);
					fwc4.closeFinally();
				} catch (IOException ioe) {ioe.printStackTrace();}
				// You can't get the blob key until the file is finalized
				BlobKey blobKeyRosieCSV4 = fileService.getBlobKey(csvFile);
				log.info("csv blob storage key is:" + blobKeyRosieCSV4.getKeyString());
				csvUrls[i-1] = blobKeyRosieCSV4.getKeyString();
				break;
				
				case 5: csvFile = fileRosieCSV6FamilyAssistanceProvided;
				log.info("closing out file 6");
				try {
					//locked set to true
					FileWriteChannel fwc6 = fileService.openWriteChannel(csvFile, true);
					fwc6.closeFinally();
				} catch (IOException ioe) {ioe.printStackTrace();}
				// You can't get the blob key until the file is finalized
				BlobKey blobKeyRosieCSV6 = fileService.getBlobKey(csvFile);
				log.info("csv blob storage key is:" + blobKeyRosieCSV6.getKeyString());
				csvUrls[i-1] = blobKeyRosieCSV6.getKeyString();
				break;
				default: break;  		
			}
		}
		return csvUrls;
	}
	
	void makeAllFileWritersFromPreexistingFiles(HUDXML3Blob blob,  BlobKey hudXMLURL) {
		//make the 5 csv files
		GregorianCalendar gc = new GregorianCalendar();
		SimpleDateFormat dateFormat = new SimpleDateFormat("MM_dd_yyyy'T'HH_mm_ssZ");
		String date = dateFormat.format(gc.getTime()).toString();

		for (int i=1; i<6; i++) { 
			switch (i) {
				case 1:  
					if (blob.getCsvUrl1() != null && !(blob.getCsvUrl1().isEmpty())) {
						fileRosieCSV1FamilyInformation = new AppEngineFile(blob.getCsvUrl1()); 
						//fileService.getBlobFile(new BlobKey(blob.getCsvUrl1()));
					} 
					else {
						try {
							log.info("making database entry for csv1url");
							fileRosieCSV1FamilyInformation = fileService.createNewBlobFile("text/plain", "RosieCSV1FamilyInformation" + "-" + date + ".csv");
							//persist the path
							String path = fileRosieCSV1FamilyInformation.getFullPath();
							blob.setCsvUrl1(path);
							HUDXML3BlobService hxbs = new HUDXML3BlobService();
							hxbs.updateHUDXML3Blob(blob);
						} catch (IOException e) {
							log.info("Problem creating fileRosieCSV1FamilyInformation: " + e.getMessage());
						}	
					}
					log.info("making blob file 1");
					break;
				case 2: 
					if (blob.getCsvUrl2() != null && !(blob.getCsvUrl2().isEmpty())) {
						fileRosieCSV2FamilyMembers = new AppEngineFile(blob.getCsvUrl2()); 
						//fileService.getBlobFile(new BlobKey(blob.getCsvUrl2()));
					}
					else {
						try {
							log.info("making database entry for csv2url");
							fileRosieCSV2FamilyMembers = fileService.createNewBlobFile("text/plain", "RosieCSV2FamilyMembers" + "-" + date + ".csv");
							//persist the path
							String path = fileRosieCSV2FamilyMembers.getFullPath();
							blob.setCsvUrl2(path);
							HUDXML3BlobService hxbs = new HUDXML3BlobService();
							hxbs.updateHUDXML3Blob(blob);
						} catch (IOException e) {
							log.info("Problem creating fileRosieCSV2FamilyMembers: " + e.getMessage());
						}
					}
					log.info("making blob file 2");
					break;
				case 3:
					if (blob.getCsvUrl3() != null && !(blob.getCsvUrl3().isEmpty())) {
						fileRosieCSV3FamilyIncomeAndExpense = new AppEngineFile(blob.getCsvUrl3()); 
						//fileService.getBlobFile(new BlobKey(blob.getCsvUrl3()));
					}
					else {
						try {
							log.info("making database entry for csv3url");
							fileRosieCSV3FamilyIncomeAndExpense = fileService.createNewBlobFile("text/plain", "RosieCSV3FamilyIncomeAndExpense" + "-" + date+ ".csv");
							//persist the path
							String path = fileRosieCSV3FamilyIncomeAndExpense.getFullPath();
							blob.setCsvUrl3(path);
							HUDXML3BlobService hxbs = new HUDXML3BlobService();
							hxbs.updateHUDXML3Blob(blob);
						} catch (IOException e) {
							log.info("Problem creating fileRosieCSV3FamilyIncomeAndExpense: " + e.getMessage());
						}
					}
					log.info("making blob file 3");
					break;
				case 4: 
					if (blob.getCsvUrl4() != null && !(blob.getCsvUrl4().isEmpty())) {
						fileRosieCSV4MemberHealthProblems = new AppEngineFile(blob.getCsvUrl4());
						//fileService.getBlobFile(new BlobKey(blob.getCsvUrl4()));
					}
					else {
						try {
							log.info("making database entry for csv4url");
							fileRosieCSV4MemberHealthProblems = fileService.createNewBlobFile("text/plain", "RosieCSV4MemberHealthProblems" + "-" + date+ ".csv");
							//persist the path
							String path = fileRosieCSV4MemberHealthProblems.getFullPath();
							blob.setCsvUrl4(path);
							HUDXML3BlobService hxbs = new HUDXML3BlobService();
							hxbs.updateHUDXML3Blob(blob);
						} catch (IOException e) {
							log.info("Problem creating fileRosieCSV4MemberHealthProblems: " + e.getMessage());
						}
					}
					log.info("making blob file 4");
					break;
				case 5: 
					if (blob.getCsvUrl6() != null && !(blob.getCsvUrl6().isEmpty())) {
						fileRosieCSV6FamilyAssistanceProvided = new AppEngineFile(blob.getCsvUrl6());
						//fileService.getBlobFile(new BlobKey(blob.getCsvUrl6()));
					}
					else {
						try {
							log.info("making database entry for csv6url");
							fileRosieCSV6FamilyAssistanceProvided = fileService.createNewBlobFile("text/plain", "RosieCSV6FamilyAssistanceProvided" + "-" + date+ ".csv");
							//persist the path
							String path = fileRosieCSV6FamilyAssistanceProvided.getFullPath();
							blob.setCsvUrl6(path);
							HUDXML3BlobService hxbs = new HUDXML3BlobService();
							hxbs.updateHUDXML3Blob(blob);
						} catch (IOException e) {
							log.info("Problem creating fileRosieCSV6FamilyAssistanceProvided: " + e.getMessage());
						}
					}
					log.info("making blob file 6");
					break;
				default: break;  
			}
		}
	}
	
	void makeHeaders(BlobKey hudXMLURL) {
		// returns the blobstore urls for all of the to-be converted Rosie csv files, with headers  
		// the input string is the url of the uploaded input HUD XML file it needs to convert 
		//Create the five Rosie CSV files, empty at first
		HUDXML3BlobService hxbs = new HUDXML3BlobService();
		makeAllFileWritersFromPreexistingFiles(hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString()), hudXMLURL);
		//createRosieCSVSet();
		//XMLStreamReader xmlr = null;
		//do actual HUD XML -> Rosie CSV conversion here 
		//read hud xml blob file into memory
		//Start a GAE BlobstoreService session

		//print the headers
		log.info("writing out file1 header" );
		try {
			FileWriteChannel fwc1 = fileService.openWriteChannel(fileRosieCSV1FamilyInformation, false);
			PrintWriter pw1 = new PrintWriter(Channels.newWriter(fwc1, "UTF8"));
			CSVWriter writer1 = new CSVWriter(new BufferedWriter(pw1));
			writeNextAndClose(writer1, fwc1, pw1, fileRosieCSV1FamilyInformation, file1Record);
		} catch (IOException ioe) {ioe.printStackTrace();}
		//clear record
		for (int i = 0; i < file1Record.length; i++) 
			file1Record[i] = "";
		log.info("writing out file2 header" );
		
		try {		
			FileWriteChannel fwc2 = fileService.openWriteChannel(fileRosieCSV2FamilyMembers, false);
			PrintWriter pw2 = new PrintWriter(Channels.newWriter(fwc2, "UTF8"));
			CSVWriter writer2 = new CSVWriter(new BufferedWriter(pw2));
			writeNextAndClose(writer2, fwc2, pw2, fileRosieCSV2FamilyMembers, file2Record);
		} catch (IOException ioe) {ioe.printStackTrace();}

		//clear record
		for (int i = 0; i < file2Record.length; i++) 
			file2Record[i] = "";
		log.info("writing out file3 header" );
		try {
			FileWriteChannel fwc3 = fileService.openWriteChannel(fileRosieCSV3FamilyIncomeAndExpense, false);
			PrintWriter pw3 = new PrintWriter(Channels.newWriter(fwc3, "UTF8"));
			CSVWriter writer3 = new CSVWriter(new BufferedWriter(pw3));
			writeNextAndClose(writer3, fwc3, pw3, fileRosieCSV3FamilyIncomeAndExpense, file3Record);
		} catch (IOException ioe) {ioe.printStackTrace();}
		
		//clear record
		for (int i = 0; i < file3Record.length; i++) 
			file3Record[i] = "";
		log.info("writing out file4 header" );
		try {
			FileWriteChannel fwc4 = fileService.openWriteChannel(fileRosieCSV4MemberHealthProblems, false);
			PrintWriter pw4 = new PrintWriter(Channels.newWriter(fwc4, "UTF8"));
			CSVWriter writer4 = new CSVWriter(new BufferedWriter(pw4));
			writeNextAndClose(writer4, fwc4, pw4, fileRosieCSV4MemberHealthProblems, file4Record);
		} catch (IOException ioe) {ioe.printStackTrace();}

		//clear record
		for (int i = 0; i < file4Record.length; i++) 
			file4Record[i] = "";
		log.info("writing out file6 header" );
		try {
			FileWriteChannel fwc6 = fileService.openWriteChannel(fileRosieCSV6FamilyAssistanceProvided, false);
			PrintWriter pw6 = new PrintWriter(Channels.newWriter(fwc6, "UTF8"));
			CSVWriter writer6 = new CSVWriter(new BufferedWriter(pw6));
			writeNextAndClose(writer6, fwc6, pw6, fileRosieCSV6FamilyAssistanceProvided, file6Record);
		} catch (IOException ioe) {ioe.printStackTrace();}

		//clear record
		for (int i = 0; i < file6Record.length; i++) 
			file6Record[i] = "";
	}
	
	boolean recordExists(String id) {
		boolean result = false;
		HUDXML3BlobService hxbs = new HUDXML3BlobService();
		HUDXML3Blob blob = hxbs.getHUDXML3Blob(id);
		if (blob != null) {
			if (blob.getCsvUrl1Indexed() == 1) {
				result = true;
			}
		}
		return result;
	}

	String [] getNextIndexToProcess(String id, int fileNumber) {
		log.info("inside getNextIndexToProcess.  id is: " + id + " fileNumber is: " + fileNumber);
		String[] nextIndexRecord = new String[2];
		//get next index for file from persistence
		EntityManager em;
		em = EMF.get().createEntityManager();
		String file_index_column = "file" + fileNumber + "index";
		//Query q = em.createQuery ("SELECT rosie.id, rosie." + file_index_column + " FROM RosieIndexes rosie WHERE rosie.xmlBlobId = :xmlBlobId AND rosie." + file_index_column + " = (SELECT MIN(rosie." + file_index_column + ") FROM RosieIndexes rosie) AND rosie.used = 0");
		//Query q = em.createQuery ("SELECT rosie FROM RosieIndexes rosie WHERE rosie.xmlBlobId = :xmlBlobId AND rosie." + file_index_column + " = (SELECT MIN(rosie." + file_index_column + ") FROM RosieIndexes rosie) AND rosie.used = 0");
		
		//This needs to be improved, but it works fine.  It just doesn't go in true ascending order
		Query q = em.createQuery ("SELECT rosie FROM RosieIndexes rosie WHERE rosie.xmlBlobId = :xmlBlobId AND rosie." + file_index_column + " = (SELECT MIN(rosie." + file_index_column + ") FROM RosieIndexes rosie WHERE rosie.used = 0)");
		//Doesn't work; use the criteria builder to get true ascending
		//Query q = em.createQuery ("SELECT rosie FROM RosieIndexes rosie WHERE rosie.xmlBlobId = :xmlBlobId AND rosie.used = 0 ORDER BY SQL('? ASC LIMIT 1',  rosie." + file_index_column + ")"); 
		//CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
		//CriteriaQuery<RosieIndexes> criteriaQuery = criteriaBuilder.createQuery(RosieIndexes.class);
		//Root<RosieIndexes> blobs = criteriaQuery.from(RosieIndexes.class);
		//criteriaQuery.select(blobs);
		//log.info("searching for RosieIndexes record with xmlBlobId: " + id);
		//Predicate predicate = criteriaBuilder.equal(blobs.get(RosieIndexes_.xmlBlobId), new BigInteger(id));
		//criteriaQuery.where(predicate);
		//TypedQuery<RosieIndexes> typedQuery = em.createQuery(criteriaQuery);
		
		if (id != null) {
			q.setParameter ("xmlBlobId",(new BigInteger(id)));
			RosieIndexes r = null;
			try {
				@SuppressWarnings("unchecked")
				List<RosieIndexes> resultList = q.getResultList();
				for (RosieIndexes i : resultList) {
					log.info("RosieIndexes record ID returned in getNextIndexToProcess is: " + i.getId());
				}
				r = (RosieIndexes) q.getSingleResult();
			} 
			catch (NoResultException nr) {
				log.info("no results from query.  wait, then try again.");
				try {
					  Thread.sleep(70);
					  r = (RosieIndexes) q.getSingleResult();
				}  
				catch (NoResultException nr1) {
					log.info("no results from query.  wait, then try again.");
					try {
						Thread.sleep(200);
						r = (RosieIndexes) q.getSingleResult();
					} 
					catch (NoResultException nr2) {
						log.info("no results from query, even after waiting twice.");
						nextIndexRecord[0] = "-1";
						nextIndexRecord[1] = "-1";
						return nextIndexRecord;
						
					}
					catch (InterruptedException ie) {
						log.info("Timer issue: ");
						ie.printStackTrace();
					}
				}
				catch (InterruptedException ie) {
					log.info("Timer issue: ");
					ie.printStackTrace();
				}
			}
			
			String idResult = r.getId();
			String indexResult = null;
			if (fileNumber == 1) {
				indexResult = r.getFile1index();
			}
			else if (fileNumber == 2) {
				indexResult = r.getFile2index();
			}
			else if (fileNumber == 3) {
				indexResult = r.getFile3index();
			}
			else if (fileNumber == 4) {
				indexResult = r.getFile4index();
			}
			else if (fileNumber == 6) {
				indexResult = r.getFile6index();
			}
			nextIndexRecord[0] = idResult;
			nextIndexRecord[1] = indexResult;
			log.info("getNextIndexToProcess got next id: " + idResult + " and next index: " + indexResult);
			if (nextIndexRecord[0] == null || nextIndexRecord[0].isEmpty()) {
				nextIndexRecord[0] = "-1";
				nextIndexRecord[1] = "-1";
			}
		}
		return nextIndexRecord;
	}
	
	InputStream openXMLStream(BlobKey blobKeyHUDXML) {
		InputStream inputStream = null;
		try {	
			log.info("Using BlobstoreInputStream");
			inputStream = new BlobstoreInputStream(blobKeyHUDXML);
		} catch (Exception e) {
			log.info("Error reading from BlobstoreInputStream:" + e.toString());
		}			

		//unzip the input stream
		InputStream zis = null;
		try {
			zis = new BufferedInputStream(new GZIPInputStream(inputStream));
		} catch (IOException e) {
			log.info("Problem creating BufferedInputStream from GZIP" + e.toString());
		}
		return zis;
	}
	
	@SuppressWarnings("unchecked")
	boolean finishedIndexing(String id, int fileNumber) {
		boolean status = false;
		//get status for file from persistence
		List<Integer> results = null;
		EntityManager em;
		em =EMF.get().createEntityManager();
		String file_column = "csvUrl" + fileNumber + "Indexed";
		Query q = em.createQuery ("SELECT hblob." + file_column + " FROM HUDXML3Blob AS hblob WHERE hblob.id = :id");
	//"SELECT pc.clientKey FROM PathClient AS pc WHERE pc.ssn =  :ssn AND pc.inactive = '0'");

		if (id != null) {
			q.setParameter ("id", id);
			try {
				Thread.sleep(500);
				results = q.getResultList();
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		if (results.get(0) == 1) {
			status = true;
		} else {log.warn("reindexing csv file " + fileNumber +", because the index wasn't detected as a '1'");}
		return status;
	}
	@SuppressWarnings("unchecked")
	boolean finishedConverting(String id, int fileNumber) {
		boolean status = false;
		//get status for file from persistence
		List<Integer> results = null;
		EntityManager em;
		em =EMF.get().createEntityManager();		
		String file_column = "csvUrl" + fileNumber + "Completed";
		Query q = em.createQuery ("SELECT hblob." + file_column + " FROM HUDXML3Blob AS hblob WHERE hblob.id = :id");
		if (id != null) {
			q.setParameter ("id", id);
			//@SuppressWarnings("unchecked")
			results = q.getResultList();
		}
		if (results.get(0) == 1) {
			status = true;
		}
		return status;
	}
	
	void convertHUDXMLToRosieCSV1(String id, BlobKey hudXMLURL) {
		//check if finished indexing
		if (finishedIndexing(id, 1)) {
			log.info("it was determined that file 1 had finished indexing");
			// pick an unprocessed index
			boolean keepRunning = true;
			while (keepRunning) {
				String[] nextIndexRecord = getNextIndexToProcess(id, 1);
				String nextIndex = nextIndexRecord[1];
				if (nextIndex == null || nextIndex == "-1") {
					keepRunning = false;
				} else {
					processNextCSV1(nextIndex, hudXMLURL);
					//mark the nextIndex as processed
					RosieIndexesService ris = new RosieIndexesService();
					RosieIndexes ri = ris.getRosieIndexes(nextIndexRecord[0]);
					ri.setUsed(1);
					ris.updateRosieIndexes(ri);
				}
			}
		} 
		//else index it
		else {
			indexCSV1(id, hudXMLURL);
			//recurse this method to process remaining indexes, and remember to save/write as you go
			convertHUDXMLToRosieCSV1(id, hudXMLURL);
		}
	}
	
	void indexCSV1(String id, BlobKey hudXMLURL) {
		log.info("indexing file 1");
		InputStream zis = openXMLStream(hudXMLURL);
		
		//JAXBContext jc = null;

		//Use StAX to get all the Program Enrollments (SiteServiceParticipations) one for each File 1 Family Information Record 
		XMLStreamReader xmlr0 = null;

		try {
			// set up a StAX reader to find all the indexes within the XML, which later key the CSV file
			xmlr0 = xmlif.createXMLStreamReader(zis);
		
		
			//Use StAX to retrieve serviceID (proxy for agencyID) from the SiteService
			// get all the indexes and put them into the db
			try {
				while(xmlr0.hasNext()) {
					// Get and check each SiteService for the right SiteServiceID
					xmlr0.next();
					if (xmlr0.isStartElement()) {
						// System.out.println("local name is: " + xmlr.getLocalName());
						if (xmlr0.getLocalName().equals("SiteServiceParticipation") && xmlr0.isStartElement()) {
							xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "SiteServiceParticipation");
							// get the SiteServiceParticipationID element following soon after this element
							while(xmlr0.hasNext()) {
								xmlr0.next();
								if (xmlr0.isStartElement()) {
									if (xmlr0.getLocalName().equals("IDNum")) {
										xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "IDNum");
										//save this index in db for later sequential processing
										String index = xmlr0.getElementText();
										RosieIndexes r = new RosieIndexes();
										r.setXmlBlobId(new BigInteger(id));
										r.setFile1index(index);
										RosieIndexesService ris = new RosieIndexesService();
										//log.info("file 1 being indexed");
										ris.createRosieIndexes(r);
										break;
									}
								}
							}
						} 	
					}
				}
				//mark this file as indexed
				HUDXML3BlobService hxbs = new HUDXML3BlobService();
				log.info("hudXMLURL.getKeyString() for blob being searched for" + hudXMLURL.getKeyString());
				HUDXML3Blob blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
				log.info("setting CsvUrl1Indexed to '1'");
				blob.setCsvUrl1Indexed(1);
				hxbs.updateHUDXML3Blob(blob);
			} catch (Exception e) {
					log.info("XML Stream exception caught: " + e.toString()); e.printStackTrace();
			}
			xmlr0.close();
			zis.close();
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
	}
	
	void convertHUDXMLToRosieCSV2(String id, BlobKey hudXMLURL) {

		//check if finished indexing
		if (finishedIndexing(id, 2)) {
			log.info("it was determined that file 2 had finished indexing");
			// pick an unprocessed index
			boolean keepRunning = true;
			while (keepRunning) {
				String[] nextIndexRecord = getNextIndexToProcess(id, 2);
				String nextIndex = nextIndexRecord[1];
				if (nextIndex == null || nextIndex == "-1") {
					keepRunning = false;
				} else {
					processNextCSV2(nextIndex, hudXMLURL);
					//mark nextIndex as processed
					RosieIndexesService ris = new RosieIndexesService();
					RosieIndexes ri = ris.getRosieIndexes(nextIndexRecord[0]);
					ri.setUsed(1);
					ris.updateRosieIndexes(ri);
				}
			}
		} 
		//else index it
		else {
			indexCSV2(id, hudXMLURL);
			//recurse this method to process remaining indexes, and remember to save/write as you go
			convertHUDXMLToRosieCSV2(id, hudXMLURL);
		}
	}

	void indexCSV2(String id, BlobKey hudXMLURL) {
		log.info("indexing file 2");
		InputStream zis = openXMLStream(hudXMLURL);
		
		//JAXBContext jc = null;

		//Use StAX to get all the Households one for each File 2  Record 
		XMLStreamReader xmlr0 = null;

		try {
			// set up a StAX reader
			xmlr0 = xmlif.createXMLStreamReader(zis);
			//Using StAX to get all the household ids for File 2 Household Membership Generation
			try {
				while(xmlr0.hasNext()) {
					// Get and check each SiteService for the right SiteServiceID
					xmlr0.next();
					if (xmlr0.isStartElement()) {
						// System.out.println("local name is: " + xmlr.getLocalName());
						if (xmlr0.getLocalName().equals("Household") && xmlr0.isStartElement()) {
							xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "Household");
							// get the HouseholdID element following soon after this element
							while(xmlr0.hasNext()) {
								xmlr0.next();
								if (xmlr0.isStartElement()) {
									if (xmlr0.getLocalName().equals("IDStr")) {
										xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "IDStr");
										//save this index in db for later sequential processing
										String index = xmlr0.getElementText();
										RosieIndexes r = new RosieIndexes();
										r.setXmlBlobId(new BigInteger(id));
										r.setFile2index(index);
										RosieIndexesService ris = new RosieIndexesService();
										//log.info("file 2 being indexed");
										ris.createRosieIndexes(r);
										break;
									} 
									else if (xmlr0.getLocalName().equals("IDNum")) {
										xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "IDNum");
										//save this index in db for later sequential processing
										String index = xmlr0.getElementText();
										RosieIndexes r = new RosieIndexes();
										r.setXmlBlobId(new BigInteger(id));
										r.setFile2index(index);
										RosieIndexesService ris = new RosieIndexesService();
										//log.info("file 2 being indexed");
										ris.createRosieIndexes(r);
										break;
									}
								}
							}
						} 	
					}
				}
				//mark this file as indexed
				HUDXML3BlobService hxbs = new HUDXML3BlobService();
				log.info("hudXMLURL.getKeyString() for blob being searched for" + hudXMLURL.getKeyString());
				HUDXML3Blob blob = hxbs.getHUDXML3BlobFromXMLKey(hudXMLURL.getKeyString());
				blob.setCsvUrl2Indexed(1);
				hxbs.updateHUDXML3Blob(blob);	
			} catch (Exception e) {
				log.info("XML Stream exception caught: " + e.toString());
			}
			xmlr0.close();
			zis.close();
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
	}
	
	void convertHouseholdtoRosieCSV2(Household household, BlobKey hudXMLURL) {
		// populate ROSIE CSV file 2 "Household Information" with each person that is a household member
		// each household member represents one record in Rosie CSV #2
		log.info("inside convertHouseholdtoRosieCSV()");

		//Use JAXB to get the Head of Household's ID
		BigInteger headOfHouseholdID = household.getHeadOfHouseholdID().getIDNum();
		//headOfHouseholdIDList = (NodeList) xpath.evaluate("/Sources/Source/Export/Household[HouseholdID/IDStr/text() = \"" + hid+"\"]/HeadOfHouseholdID/IDNum", doc, XPathConstants.NODESET);
		//log.info("Exception while getting the head of household's id for household " + hid + ": " + e.toString());
		String sSNFamilyHeadCSV2 = null;
		 
		//look up the head of household's SSN
		sSNFamilyHeadCSV2 = getHeadOfHouseholdSSNFromHousehold(household, hudXMLURL);

		log.info("Placing \'" + sSNFamilyHeadCSV2 + "\' into file2Record[3]");
		file2Record[3] = sSNFamilyHeadCSV2;
		//No mapping yet for RecordType_2
		file2Record[4] = "";
		
		//Get the Agency Code 
		//First, get the household's id, and look up which program participation that is part of
		log.info("Attempting to get Agency Code for File 2");
		
		//String hid = household.getHouseholdID().getIDStr();
		//log.info("Exception trying to get household IDStr: " + e.toString());
		//use this household to look up each of that household's members and complete Rosie CSV 2 "Family Members"
		
		String hh_id;
		if (household.getHouseholdID().getIDStr() != null) {
			hh_id = household.getHouseholdID().getIDStr();
		} else if (household.getHouseholdID().getIDNum() != null) {
			hh_id = household.getHouseholdID().getIDNum().toString();
		} else {log.info("couldn't get household id in convertHouseholdtoRosieCSV2"); hh_id = "";}
		log.info("calling processHouseholdMembersToRosie for household: " + hh_id);
		processHouseholdMembersToRosie(household, hudXMLURL);
		
		//resetting the record data for just the head of household, since it was written out/cleared
		file2Record[3] = sSNFamilyHeadCSV2;
		// Because, in this particular XML, the Head Of Household is not listed as a household member (which it should be, also process that person separately.
		processMembersAndHeadsOfHouseholds(household, headOfHouseholdID, hudXMLURL);
	}

	void processNextCSV1(String nextIndex, BlobKey hudXMLURL) {
		log.info("inside processNextCSV1.  nextIndex is: " + nextIndex);
		
		//Use StAX to get Agencies
		JAXBContext jc = null;
		try {
			jc = JAXBContext.newInstance(SiteServiceParticipation.class);
		} catch (JAXBException e) {
			log.info("JAXB SiteServiceParticipation.class context problem: " + e.toString());
			e.printStackTrace();
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling SiteServiceParticipation context: " + e.toString());
			e.printStackTrace();
		}
		
		try {
 			InputStream zis = openXMLStream(hudXMLURL);
			XMLStreamReader xmlr0 = null;
			// set up a StAX reader to find the nextIndex within the XML
			xmlr0 = xmlif.createXMLStreamReader(zis);
			//boolean foundTheSsp = false;
			while(xmlr0.hasNext()) {
				//Get and check each SiteService for the right SiteServiceID
				xmlr0.next();
				if (xmlr0.isStartElement()) {
					//System.out.println("local name is: " + xmlr.getLocalName());
					if (xmlr0.getLocalName().equals("SiteServiceParticipation")) {
						xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "SiteServiceParticipation");
						//there's no way to look ahead, unless we use the stax iterator api
						JAXBElement<SiteServiceParticipation> siteServiceParticipationJAXB = unmarshaller.unmarshal(xmlr0, SiteServiceParticipation.class);
						SiteServiceParticipation siteServiceParticipationElement = (SiteServiceParticipation)siteServiceParticipationJAXB.getValue();
						if (nextIndex.equals(siteServiceParticipationElement.getSiteServiceParticipationID().getIDNum().toString())) {
							//do conversion with JAXB now
							convertSiteServiceParticipationtoRosieCSV1(siteServiceParticipationElement, hudXMLURL);
							break;
						}
					}
				}
			}
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString()); e.printStackTrace();
			e.printStackTrace();
		}
	}
	
	void processNextCSV2(String nextIndex, BlobKey hudXMLURL) {
		log.info("inside processNextCSV2.  nextIndex is: " + nextIndex);
		
		//Use StAX to get Households
		JAXBContext jc = null;
		try {
			jc = JAXBContext.newInstance(Household.class);
		} catch (JAXBException e) {
			log.info("JAXB Household.class context problem: " + e.toString());
			e.printStackTrace();
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Household context: " + e.toString());
			e.printStackTrace();
		}
		
		try {
 			InputStream zis = openXMLStream(hudXMLURL);
			XMLStreamReader xmlr0 = null;
			// set up a StAX reader to find the nextIndex within the XML
			xmlr0 = xmlif.createXMLStreamReader(zis);			
			while(xmlr0.hasNext()) {
				//Get each household
				xmlr0.next();
				if (xmlr0.isStartElement()) {
					//log.info("name is:" + xmlr1.getLocalName());
					if (xmlr0.getLocalName().equals("Household")) {
						xmlr0.require(XMLStreamConstants.START_ELEMENT, null, "Household");
						JAXBElement<Household> householdJAXB = unmarshaller.unmarshal(xmlr0,Household.class);
						Household householdElement = (Household)householdJAXB.getValue();
						if (householdElement.getHouseholdID().getIDStr() != null) {
							if ((householdElement.getHouseholdID().getIDStr()).equals(nextIndex)) {
								log.info("Now converting household: " + householdElement.getHouseholdID().getIDStr());
								try {
									convertHouseholdtoRosieCSV2(householdElement, hudXMLURL);
								} catch (Exception e) {log.info("Problem converting household: " + householdElement.getHouseholdID().getIDStr() + " " + e.toString()); e.printStackTrace();}
								break;
							}
						}
						else if (householdElement.getHouseholdID().getIDNum() != null) {
							if ((householdElement.getHouseholdID().getIDNum().toString()).equals(nextIndex)) {
								log.info("Now converting household: " + householdElement.getHouseholdID().getIDNum());
								try {
									convertHouseholdtoRosieCSV2(householdElement, hudXMLURL);
								} catch (Exception e) {log.info("Problem converting household: " + householdElement.getHouseholdID().getIDNum() + " " + e.toString()); e.printStackTrace();}
								break;
							}
						}
						else {log.info("Couldn't get household ID in processNextCSV2");}
					}
				}
			}
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
			e.printStackTrace();
		}
	}
	
	void processFundingSourceServiceEvent(Household household, BlobKey hudXMLURL) {
		//log.info("inside processFundingSourceServiceEvent()");
		String transactionDateCSV6 = null;
		String transactionTimeCSV6 = null;
		String sSNFamilyHeadCSV6 = null;
		String AssistanceProvidedAmountCSV6 = null;
		String familyMemberNumberCSV6 = null;
		for (int i=0; i<file6Record.length; i++) {
			file6Record[i]="";
		}

		//Get all of this household's fundingSources
		//"/Sources/Source/Export/Person/SiteServiceParticipation[HouseholdID/IDNum/text() = \"" + hid + "\"]/ServiceEvent/FundingSources/FundingSource", doc, XPathConstants.NODESET));
			
		ArrayList<SiteServiceParticipation> ssps = getSiteServiceParticipationsForHousehold(household, hudXMLURL);
		if (!ssps.isEmpty()) {
			for (SiteServiceParticipation ssp : ssps) {
				for (ServiceEvent serviceEvent : ssp.getServiceEvent()) {
					for (FundingSourceServiceEvent fs : serviceEvent.getFundingSources().getFundingSource()) {
						//For each funding source, write a record.
						//Each record starts with an agency
						//Look up the service key, since we use that instead of agency
						BigInteger serviceID = getServiceIDFromSiteServiceID(ssp.getSiteServiceID(), hudXMLURL);
						if (serviceID.signum() == 1)
							file6Record[0] = serviceID.toString();

						//Get transaction Date and Time from FundingSource/FinancialAssistanceAmount/@date_effective 
						//"/FinancialAssistanceAmount/@dateEffective", fundingSource, 
						
						GregorianCalendar calendar = serviceEvent.getServiceEventEffectivePeriod().getStartDate().toGregorianCalendar();
						//log.info("Problem getting TransactionDates: " + e.toString());
						SimpleDateFormat formatOutDate = new SimpleDateFormat("MMddyy");
						SimpleDateFormat formatOutTime = new SimpleDateFormat("HHmmss");
						formatOutDate.setCalendar(calendar);
						formatOutTime.setCalendar(calendar);
		
						//SimpleDateFormat formatIn = new SimpleDateFormat("yyyy-MM-dd'T'hh':'mm':'ss");
						Date date;
						date = calendar.getTime();
						transactionDateCSV6 = formatOutDate.format(date);
						transactionTimeCSV6 = formatOutTime.format(date);
						
						//log.info("Problem formatting a date: " + e.toString());
						file6Record[1] = transactionDateCSV6;
						file6Record[2] = transactionTimeCSV6;
	
						//Get SSNFamilyHead
						sSNFamilyHeadCSV6 = getHeadOfHouseholdSSNFromHousehold(household, hudXMLURL);
						//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + headOfHouseholdID+"\"]/SocialSecurityNumber/Unhashed/text()", doc, XPathConstants.NODESET)).item(0).getTextContent();
						//log.info("Exception trying to get SSN for head of household: " + e.toString());
						file6Record[3] = sSNFamilyHeadCSV6;
						
						//Get family member id from Person element ancestor
						Person person = getPersonFromSiteServiceParticipation(ssp, hudXMLURL);
						familyMemberNumberCSV6 = person.getPersonID().getIDNum().toString();
						file6Record[5] = familyMemberNumberCSV6;
	
						//Get AssistanceProvidedCode
						//No corresponding entry in HUD XML
						//file6Record[6] = AssistanceProvidedCodeCSV6;
	
						//Get AssistanceProvidedDescription
						//No corresponding entry in HUD XML
						//file6Record[7] = AssistanceProvidedDescriptionCSV6;
	
						//Get AssistanceProvidedAmount  
						AssistanceProvidedAmountCSV6 = fs.getFinancialAssistanceAmount().toString();
						//"FinancialAssistanceAmount/text()", fundingSource, XPathConstants.NODESET)).item(0).getTextContent();
						//log.info("Problem getting a FinancialAssistanceAmount: " + e.toString());
						file6Record[8] = AssistanceProvidedAmountCSV6;
						
						try {
							FileWriteChannel fwc6 = fileService.openWriteChannel(fileRosieCSV6FamilyAssistanceProvided, false);
							PrintWriter pw6 = new PrintWriter(Channels.newWriter(fwc6, "UTF8"));
							CSVWriter writer6 = new CSVWriter(new BufferedWriter(pw6));
							writeNextAndClose(writer6, fwc6, pw6, fileRosieCSV6FamilyAssistanceProvided, file6Record);
						} catch (IOException ioe) {ioe.printStackTrace();}

						//clear record
						for (int i = 0; i < file6Record.length; i++) {
							file6Record[i] = "";
						}
					}
				}	
			}
		} else {
			if (household.getHouseholdID().getIDStr() != null) {
			log.info("No matching SiteServiceParticipations found for household: " + household.getHouseholdID().getIDStr());
			} else if (household.getHouseholdID().getIDNum() != null) {
				log.info("No matching SiteServiceParticipations found for household: " + household.getHouseholdID().getIDNum());
			}
		}
	}  

	void processHealthStatus(Household household, BlobKey hudXMLURL) {
		log.info("inside processHealthStatus()");
		String transactionDateCSV5;
		String transactionTimeCSV5;
		String sSNFamilyHeadCSV5;
		String familyMemberNumberCSV5;

		//Get all of this household's health statuses
		ArrayList<SiteServiceParticipation> ssps = getSiteServiceParticipationsForHousehold(household, hudXMLURL);
		for (SiteServiceParticipation siteServiceParticipation : ssps) {
			for (PersonHistorical ph : siteServiceParticipation.getPersonHistorical()) {
				for (HealthStatus healthStatus : ph.getHealthStatus()) {
					//For each health status, write a record.
					//Each record starts with a Service (would normally be agency)

					//Look up the Service Key 
					BigInteger siteServiceID = siteServiceParticipation.getSiteServiceID();
					file4Record[0] = getServiceIDFromSiteServiceID(siteServiceID, hudXMLURL).toString();

					//Get transaction Date and Time from IncomeAndSource/IncomeSourceCode@date_effective 
					GregorianCalendar calendar = healthStatus.getDateEffective().toGregorianCalendar();
					//log.info("Problem getting TransactionDates: " + e.toString());
					SimpleDateFormat formatOutDate = new SimpleDateFormat("MMddyy");
					SimpleDateFormat formatOutTime = new SimpleDateFormat("HHmmss");
					formatOutDate.setCalendar(calendar);
					formatOutTime.setCalendar(calendar);
	
					//SimpleDateFormat formatIn = new SimpleDateFormat("yyyy-MM-dd'T'hh':'mm':'ss");
					Date date;
					date = calendar.getTime();
					transactionDateCSV5 = formatOutDate.format(date);
					transactionTimeCSV5 = formatOutTime.format(date);
				
					file4Record[1] = transactionDateCSV5;
					file4Record[2] = transactionTimeCSV5;
	
					//Get SSNFamilyHead
					sSNFamilyHeadCSV5 = getHeadOfHouseholdSSNFromHousehold(household, hudXMLURL);
					//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + headOfHouseholdID+"\"]/SocialSecurityNumber/Unhashed/text()", doc, XPathConstants.NODESET)).item(0).getTextContent();
					//log.info("Exception trying to get SSN for head of household: " + e.toString());
					file4Record[3] = sSNFamilyHeadCSV5;
					file4Record[4] = sSNFamilyHeadCSV5;

					//Get family member id from Person element ancestor
					Person person = getPersonFromSiteServiceParticipation(siteServiceParticipation, hudXMLURL);
					familyMemberNumberCSV5 = person.getPersonID().getIDNum().toString();
					file4Record[5] = familyMemberNumberCSV5;

					//Get HealthProblemCode
					String healthProblemCodeCSV4;
					healthProblemCodeCSV4 = String.valueOf(healthStatus.getValue());
					file4Record[6] = healthProblemCodeCSV4;

					//Get HealthProblemDescription
					//No corresponding entry in HUD XML
					file4Record[7] = "";
					try {
						FileWriteChannel fwc4 = fileService.openWriteChannel(fileRosieCSV4MemberHealthProblems, false);
						PrintWriter pw4 = new PrintWriter(Channels.newWriter(fwc4, "UTF8"));
						CSVWriter writer4 = new CSVWriter(new BufferedWriter(pw4));
						writeNextAndClose(writer4, fwc4, pw4, fileRosieCSV4MemberHealthProblems, file4Record);
					} catch (IOException ioe) {ioe.printStackTrace();}

					//clear record
					for (int i = 0; i < file4Record.length; i++) {
						file4Record[i] = "";
					}
				}
				
			}
		}
		//"/Sources/Source/Export/Person/SiteServiceParticipation[HouseholdID/IDNum/text() = \"" + hid + "\"]/PersonHistorical/HealthStatus/text()", doc, XPathConstants.NODESET));
		//log.info("Problem getting HealthStatusList: " + e.toString());
	}

	void processHouseholdMembersToRosie(Household household, BlobKey hudXMLURL) {
		log.info("inside processHouseholdMembersToRosie()");
		//look up each family member, then write to Rosie CSV 2 "Family Members"

		//Get all household members
		if (household.getHouseholdID().getIDStr() != null) {
			log.info("Searching for family members of household with id: " + household.getHouseholdID().getIDStr());
		} else if (household.getHouseholdID().getIDNum() != null) {
			log.info("Searching for family members of household with id: " + household.getHouseholdID().getIDNum());
		}
		Members householdMembers;
		try {	
			householdMembers = household.getMembers();
			if (householdMembers.getMember().isEmpty()) {
				if (household.getHouseholdID().getIDStr() != null) {
					log.info("No household members found within householdID/IDStr: " + household.getHouseholdID().getIDStr());
				} else if (household.getHouseholdID().getIDNum() != null) {
					log.info("No household members found within householdID/IDNum: " + household.getHouseholdID().getIDNum());
				}
			} else {
				log.info("Number of household members found is: " + householdMembers.getMember().size());
			}
			for (Member member : householdMembers.getMember()) {
				//Get the family member's ID
				String familyMemberNumberCSV2 = null;
				try {
					familyMemberNumberCSV2 = String.valueOf(member.getPersonID().getIDNum());
					//familyMemberNumberCSV2 = xpath.evaluate("PersonID/IDNum/text()", familyMember);
				} catch (Exception e) {
					  log.info("Problem getting Person/IDNum: " + e.toString());
				}
				if (familyMemberNumberCSV2 == null | familyMemberNumberCSV2.isEmpty()) { 
					log.info("No PersonID/IDNum found.");
				} else log.info("It apparently found an IDNum (" + familyMemberNumberCSV2 + "), so processing on it should commence");

			    log.info("familyMemberNumberCSV2 number we're processing is: " + familyMemberNumberCSV2);
			    file2Record[5] = familyMemberNumberCSV2;

				//Get the family member's relationship to the head of household
				String relationshipCodeCSV2;
				try {
					relationshipCodeCSV2 = String.valueOf(member.getRelationshipToHeadOfHousehold().getValue());
					//relationshipCodeCSV2 = xpath.evaluate("RelationshipToHeadOfHousehold/text()", familyMember);
				} catch(Exception e) {
					log.info("Couldn't find RelationshipToHeadOfHousehold element within this Household/Member element: " + e.toString());
					relationshipCodeCSV2 = "";
				}
				file2Record[7] = relationshipCodeCSV2;

				//Then, look up each person record for each household member, using the person id
				processMembersAndHeadsOfHouseholds(household, member, hudXMLURL);
			}
		} catch (NullPointerException e) {
			 if (household.getHouseholdID().getIDStr() != null) {
				 log.info("No members in this household: " + household.getHouseholdID().getIDStr());
			 }  else if (household.getHouseholdID().getIDNum() != null) {
				 log.info("No members in this household: " + household.getHouseholdID().getIDNum());
			 }
		}
		//householdMemberList = ((NodeList)xpath.evaluate(//"/Sources/Source/Export/Household[HouseholdID/IDNum/text() = \"" + hid +//"\"]/Members/Member", doc, XPathConstants.NODESET));
		//log.info("Problem parsing household's member list: " + e.toString());
	}

	void processIncomeAndSources(Household household, PersonHistorical ph, SiteServiceParticipation ssp, BlobKey hudXMLURL) {		
		String transactionDateCSV3 = null;
		String transactionTimeCSV3 = null;
		String sSNFamilyHeadCSV3 = null;
		String familyMemberNumberCSV3 = null;
		String incomeExpenseTypeCodeCSV3 = null;
		
		//Get all of this household's income and expenses 
		if (ph.getIncomeAndSources() != null) {
			for (IncomeAndSource incomeAndSource : ph.getIncomeAndSources().getIncomeAndSource()) {
				List<Object> o = incomeAndSource.getIncomeSourceCodeAndIncomeSourceOtherAndReceivingIncomeSource();
				//For each IncomeAndSource, write a record.
				//Each record starts with an agency
				//Look up the agency code
				file3Record[0] = getServiceIDFromSiteServiceID(ssp.getSiteServiceID(), hudXMLURL).toString();
		
				for (Object item : o) {
					if (item instanceof IncomeSourceCode) {
						// get the effective date and the code
						//Get transaction Date and Time from IncomeAndSource/IncomeSourceCode@date_effective 
						///IncomeSourceCodeID/@dateEffective", incomeAndSource, XPathConstants.NODESET);
						IncomeSourceCode incomeSourceCode = (IncomeSourceCode)o;
						GregorianCalendar incomeSourceCodeDateEffectiveCalendar = incomeSourceCode.getDateEffective().toGregorianCalendar();
						SimpleDateFormat formatOutDate = new SimpleDateFormat("MMddyy");
						SimpleDateFormat formatOutTime = new SimpleDateFormat("HHmmss");
						formatOutDate.setCalendar(incomeSourceCodeDateEffectiveCalendar);
						Date date = incomeSourceCodeDateEffectiveCalendar.getTime();
						transactionDateCSV3 = formatOutDate.format(date);
						transactionTimeCSV3 = formatOutTime.format(date);
						//Get IncomeExpenseTypeCode
						incomeExpenseTypeCodeCSV3 = String.valueOf(incomeSourceCode.getValue());
						//xpath.evaluate("/IncomeSourceCode/text()", incomeAndSource, XPathConstants.NODESET)).item(0).getTextContent();
						//log.info("Problem getting income source code: " + e.toString());
						file3Record[1] = transactionDateCSV3;
						file3Record[2] = transactionTimeCSV3;
						file3Record[6] = incomeExpenseTypeCodeCSV3;
					}	
					if (o instanceof NonNegativeInteger) {
						//Get MonthlyAmount
						String monthlyAmountCSV3 = ((NonNegativeInteger)o).getValue().toString();
						//incomeSourceAmount = ((NodeList) xpath.evaluate("/IncomeSourceAmount/text()", incomeAndSource, XPathConstants.NODESET)).item(0).getTextContent();
						//log.info("Problem getting income source amount: " + e.toString());
						file3Record[7] = monthlyAmountCSV3;
					}
				}
			}
		}
		//Get SSNFamilyHead
		sSNFamilyHeadCSV3 = getHeadOfHouseholdSSNFromHousehold(household, hudXMLURL);
		//log.info("Exception trying to get SSN for head of household: " + e.toString());
		file3Record[3] = sSNFamilyHeadCSV3;
		file3Record[4] = "";
		//Get family member id from Person element ancestor
		Person person = getPersonFromSiteServiceParticipation(ssp, hudXMLURL);
		//log.info("Problem getting PersonID/IDNum: " + e.toString());
		//"/Sources/Source/Export/Person/SiteServiceParticipation[HouseholdID/IDNum/text() = \"" + hid + "\"]/PersonHistorical/IncomeAndSources/IncomeAndSource", doc, XPathConstants.NODESET));
		//log.info("Problem getting income and source list: " + e.toString());
		//log.info("Problem getting income and source list: " + e.toString()); 
		//log.info("Problem getting IncomeSourceCodeID/@dateEffective: " + e.toString()); transactionDateCSV3 = ""; transactionTimeCSV3 = "";}
		try {
			familyMemberNumberCSV3 = person.getPersonID().getIDNum().toString();
		} catch (Exception e) {log.info("Problem getting person id for CSV 3");}
		file3Record[5] = familyMemberNumberCSV3;
		try {
			FileWriteChannel fwc3 = fileService.openWriteChannel(fileRosieCSV3FamilyIncomeAndExpense, false);
			PrintWriter pw3 = new PrintWriter(Channels.newWriter(fwc3, "UTF8"));
			CSVWriter writer3 = new CSVWriter(new BufferedWriter(pw3));
			writeNextAndClose(writer3, fwc3, pw3, fileRosieCSV3FamilyIncomeAndExpense, file3Record);
		} catch (IOException ioe) {ioe.printStackTrace();}
		
		//clear record
		for (int i = 0; i < file3Record.length; i++) {
			file3Record[i] = "";
		}		
	}

	void processPersonAddress(PersonAddress personAddress) {
		log.info("inside processPersonAddress()");
		boolean isLastPermanentZIP = false;
		String isLastPermanentZIPString;
		isLastPermanentZIPString = String.valueOf(personAddress.getIsLastPermanentZIP().getValue()); 
			//xpath.evaluate("IsLastPermanentZIP", personAddress);
        //log.info("Couldn't evaluate IsLastPermanentZIP for this personAddress: " + e.toString());
		//file2Record[20] = ""; file2Record[21] = "";file2Record[22] = "";file2Record[23] = "";file2Record[24] = "";}
		if (isLastPermanentZIPString.equals("1")) {
			// put fields in Last Permanent Address columns
			isLastPermanentZIP = true;
		} else if (isLastPermanentZIPString.equals("0")) {
			// put fields in Last Permanent Address columns
			isLastPermanentZIP = false;
		} else {
			log.info("isLastPermanentZIP == neither 1 nor 0");
			isLastPermanentZIP = false;
		}
		String streetAddressCSV2;
		String apartmentCSV2;
		String cityCSV2;
		String stateCSV2;
		String zIP5CSV2;
		//Sequentially process child nodes
		streetAddressCSV2 = personAddress.getLine1().getValue(); 
		//xpath.evaluate("Line1", personAddress);
		//log.info("Couldn't process child node Line1 of personAddress: " + e.toString());
		if (isLastPermanentZIP)
			file2Record[20] = streetAddressCSV2;
		else
			file2Record[20] = "";
		apartmentCSV2 = personAddress.getLine2().getValue(); 
		//xpath.evaluate("Line2", personAddress);
		//log.info("Couldn't process child node Line2 of personAddress: " + e.toString());
		if (isLastPermanentZIP)
			file2Record[21] = apartmentCSV2;
		else
			file2Record[21] = "";
		cityCSV2 = personAddress.getCity().getValue(); 
		//xpath.evaluate("City", personAddress);	
		//log.info("Conuldn't process child node City of personAddress: " + e.toString());
		if (isLastPermanentZIP)
			file2Record[22] = cityCSV2;
		else
			file2Record[22] = "";
	
		stateCSV2 = personAddress.getState().getValue().value();
		//xpath.evaluate("State", personAddress);	

		//log.info("Couldn't process child node State of personAddress: " + e.toString());
		stateCSV2 = "";

		if (isLastPermanentZIP)
			file2Record[23] = stateCSV2;
		else
			file2Record[23] = "";

		zIP5CSV2 =  personAddress.getZIPCode().getValue();
		//xpath.evaluate("ZIPCode", personAddress);

   	    //log.info("Couldn't process child node ZIPCode of personAddress: " + e.toString());
		if (isLastPermanentZIP)
			file2Record[24] = zIP5CSV2;
		else
			file2Record[24] = "";
	}	

	void processPersonAddressHeadOfHousehold(String headOfHouseholdID, boolean isLastPermanent, PersonAddress personAddress) {
		log.info("inside processPersonAddressHeadOfHousehold()");
		String streetAddressCSV1 = null;
		String apartmentCSV1 = null;
		String cityCSV1 = null;
		String stateCSV1 = null;
		String zIP5CSV1 = null;

		//Sequentially process child nodes

		try {	
			streetAddressCSV1 = personAddress.getLine1().getValue();
			//streetAddressCSV1 = xpath.evaluate("Line1", personAddress);
		} catch(Exception e) {
			  log.info("Couldn't find Line1 element within this PersonAddress: " + e.toString());
		}				
		if (isLastPermanent)
			file1Record[5] = streetAddressCSV1;
		else
			file1Record[10] = streetAddressCSV1;

		try {	
			apartmentCSV1 = personAddress.getLine2().getValue();
			//apartmentCSV1 = xpath.evaluate("Line2", personAddress);
		} catch(Exception e) {
			  log.info("Couldn't find Line2 element within this PersonAddress: " + e.toString());
		}							
		if (isLastPermanent)
			file1Record[6] = apartmentCSV1;
		else
			file1Record[11] = apartmentCSV1;

		try {	
			cityCSV1 = personAddress.getCity().getValue();
			//cityCSV1 = xpath.evaluate("City", personAddress);	
		} catch(Exception e) {
			  log.info("Couldn't find City element within this PersonAddress: " + e.toString());
		}								
		if (isLastPermanent)
			file1Record[7] = cityCSV1;
		else
			file1Record[12] = cityCSV1;

		try {				
			stateCSV1 = personAddress.getState().getValue().value();
			//stateCSV1 = xpath.evaluate("State", personAddress);	
		} catch(Exception e) {
			  log.info("Couldn't find State element within this PersonAddress: " + e.toString());
		}								
		if (isLastPermanent)
			file1Record[8] = stateCSV1;
		else
			file1Record[13] = stateCSV1;

		try {
			zIP5CSV1 = personAddress.getZIPCode().getValue();
			//zIP5CSV1 = xpath.evaluate("ZIPCode", personAddress);	
		} catch(Exception e) {
			  log.info("Could not locate a ZIPCode element within this PersonAddress: " + e.toString());
		}										
		if (isLastPermanent)
			file1Record[9] = zIP5CSV1;
		else
			file1Record[14] = zIP5CSV1;
	}
	
	void processPersonHistoricalForCSV2(PersonHistorical personHistorical) {
		// for first personHistorical in personHistoricalList (should ideally look through all of them)
		log.info("inside processPersonHistoricalForCSV2()");
		String familyMemberPregnantCSV2 = null;
		String familyMemberMonthsPregnantCSV2 = null;
		String familyMemberVeteranCodeCSV2 = null;
		String housingSituationCodeCSV2 = null;
		String lengthInHousingSituationCodeCSV2 = null;

		//Get the family member's pregnancy status from their Person record
			// Get first pregnancy status, should get them all
		int pregnancyStatus = -1;
		try {
			pregnancyStatus = (int)personHistorical.getPregnancy().get(0).getPregnancyStatus().getValue();
			//evaluate("Pregnancy/PregnancyStatus/text()", personHistorical, XPathConstants.NODESET)).item(0).getTextContent());
			switch (pregnancyStatus) {
				case 0: familyMemberPregnantCSV2 = "N";
				break;
				case 1: familyMemberPregnantCSV2 = "Y";
				break;
				case 8:;
				break;
				case 9:;
				break;
				default:;
				}	
			file2Record[16] = familyMemberPregnantCSV2;
		} catch (Exception e) {
			log.info("Couldn't get pregnancyStatus");
		}
		//log.info("Could not locate any Pregnancy/PregnancyStatus elements for this Person: " + e.toString());

		//Calculate the number of months pregnant from the HUD XML due date
		GregorianCalendar dueDateCalendar = null;
		GregorianCalendar conceptionDateCalendar = null;
		try {
			dueDateCalendar = personHistorical.getPregnancy().get(0).getDueDate().getValue().toGregorianCalendar();
			conceptionDateCalendar = new GregorianCalendar();
			conceptionDateCalendar = dueDateCalendar;
			//((NodeList)xpath.evaluate("Pregnancy/DueDate/text()", personHistorical, XPathConstants.NODESET)).item(0).getTextContent();
			// subtract 9 months from the due Date (conception date)
			conceptionDateCalendar.add(Calendar.MONTH, -9);
			Date conceptionDate = conceptionDateCalendar.getTime();
			// the get the difference of the current date from conception date to get months pregnant for Rosie CSV
			Date currentDate = new Date();
			int dateDiffInDays = (int)((currentDate.getTime() - conceptionDate.getTime())/ (1000 * 60 * 60 * 24));
			familyMemberMonthsPregnantCSV2 = String.valueOf(dateDiffInDays/30);
			//log.info("Could not locate any Pregancy/DueDate elements for this person: " + e.toString());
			file2Record[17] =  familyMemberMonthsPregnantCSV2;
			//No mapping yet for Need_Shelter
			file2Record[18] =  "";
		} catch (Exception e) {
			log.info("Couldn't get pregnancy");
		}
		
		//Calculate Veteran Code
		int veteranStatus = 0;
		try {
			veteranStatus =  (int) personHistorical.getVeteran().get(0).getVeteranStatus().get(0).getValue();
			familyMemberVeteranCodeCSV2 = String.valueOf(veteranStatus);
			file2Record[19] =  familyMemberVeteranCodeCSV2;
		}  catch (Exception e) {
			log.info("Couldn't get veteran status");
		}
			//Integer.valueOf(((NodeList)xpath.evaluate("Veteran/VeteranStatus/text()", personHistorical, XPathConstants.NODESET)).item(0).getTextContent());
		//log.info("Could not find VeteranStatus for this person: " + e.toString());

		// get address of family member
		//log.info("Does this personHistorical element contain anything?");
		//log.info("length of PersonHistorical child nodes is: " +personHistorical.getChildNodes().getLength());
	
		//Just handling one address per person
		PersonAddress personAddress = null;
		try {
			personAddress = personHistorical.getPersonAddress().get(0);
			processPersonAddress(personAddress);
		}  catch (Exception e) {
			log.info("Couldn't get personAddress");
		}
		//(Element)((NodeList) xpath.evaluate("PersonAddress", personHistorical, XPathConstants.NODESET)).item(0);
	    //log.info("Could not locate a PersonAddress element for this Person: " + e.toString());

		
		// get housing situation code
		try {
			housingSituationCodeCSV2 = String.valueOf(personHistorical.getHousingStatus().get(0).getValue());
			//((NodeList)xpath.evaluate("HousingStatus/text()", 	personHistorical, XPathConstants.NODESET)).item(0).getTextContent();
			//log.info("Problem getting housing status: " + e.toString());
			file2Record[25] = housingSituationCodeCSV2;
		} catch (Exception e) {
			log.info("Couldn't get HousingStatus");
		}
		// get length in housing situation code
		try {
			lengthInHousingSituationCodeCSV2 = String.valueOf(personHistorical.getLengthOfStayAtPriorResidence().get(0).getValue());
			//((NodeList)xpath.evaluate("LengthOfStayAtPriorResidence/text()", personHistorical, XPathConstants.NODESET)).item(0).getTextContent();
			//log.info("Problem getting LengthOfStayAtPriorResidence: " + e.toString());
			file2Record[26] = lengthInHousingSituationCodeCSV2;
		}	catch (Exception e) {
			log.info("Couldn't get LengthOfStayAtPriorResidence");
		}
	}
	
	BigInteger getAgencyKeyFromSiteID(BigInteger siteID, BlobKey hudXMLURL) {
		long agencyKeyLong = -1l;
		JAXBContext jc = null;
		XMLStreamReader xmlr5 = null;
		InputStream zis;
		zis = openXMLStream(hudXMLURL);
		try {
			// set up a StAX reader
			xmlr5 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
		
		try {
			// set up a StAX reader
			xmlr5 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
		//Use StAX to get Agencies
		try {
			jc = JAXBContext.newInstance(Agency.class);
		} catch (JAXBException e) {
			log.info("JAXB Agency.class context problem: " + e.toString());
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Agency context: " + e.toString());
		}
		try {
			while(xmlr5.hasNext()) {
				//Get each agency
				xmlr5.next();
				if (xmlr5.isStartElement()) {
					if (xmlr5.getLocalName().equals( "Agency")) {
						xmlr5.require(XMLStreamConstants.START_ELEMENT, null, "Agency");
						JAXBElement<Agency> agencyJAXB = unmarshaller.unmarshal(xmlr5, Agency.class);
						Agency agencyElement = (Agency)agencyJAXB.getValue();
						//iterate through all the sites in this agency
						for (TSite site : agencyElement.getSite()) {
							//Get the Site Key  from this Agency
							//This is tricky, because of JAXB's automated model creator
							for (Serializable content : site.getContent()) {
							//for (JAXBElement<?> content : site.getContent()) {
								if (content instanceof JAXBElement<?>) {
									//How do I know this is key
									JAXBElement<?> contentJAXBElement = (JAXBElement<?>)content;
									if (contentJAXBElement.getName().getLocalPart().equals("Key")) {
										BigInteger siteKey = (BigInteger) ((JAXBElement<?>)content).getValue();
										if (siteKey.equals(siteID)) {
											agencyKeyLong = agencyElement.getKey().longValue();
											break;
										}
									}
								}
							}
						} 	
					} else log.info("Problem getting AgencyKey containing SiteID: " + siteID);			
				}
			}
			xmlr5.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
		}
		BigInteger agencyKey = BigInteger.valueOf(agencyKeyLong);
		return agencyKey;
		//xpath.evaluate("/Sources/Source/Export/Agency[Site/Key/text() = \"" + siteID + "\"]/Key/text()", doc, XPathConstants.NODESET);
	}
	
	BigInteger getServiceIDFromSiteServiceID(BigInteger siteServiceID, BlobKey hudXMLURL) {
		//Instead, just get the Service's key directly from the SiteService and use that 
		//as the Agency Key
		BigInteger serviceKey = null;
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);
		XMLStreamReader xmlr6  = null;

		try {
			// set up a StAX reader
			xmlr6 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
		
		//Use StAX to retrieve serviceID (proxy for agencyID) from the SiteService
		try {
			jc = JAXBContext.newInstance(SiteService.class);
		} catch (JAXBException e) {
			log.info("JAXB SiteService.class context problem: " + e.toString());
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling SiteService context: " + e.toString());
		}
		try {
			while(xmlr6.hasNext()) {
				//Get and check each SiteService for the right SiteServiceID
				xmlr6.next();
				if (xmlr6.isStartElement()) {
					if (xmlr6.getLocalName().equals("SiteService")) {
						xmlr6.require(XMLStreamConstants.START_ELEMENT, null, "SiteService");
						JAXBElement<SiteService> siteServiceJAXB = unmarshaller.unmarshal(xmlr6, SiteService.class);
						SiteService siteServiceElement = (SiteService)siteServiceJAXB.getValue();
						//Get the ServiceID from this Service
						if (siteServiceElement.getKey().equals(siteServiceID)) {
							serviceKey = siteServiceElement.getServiceID();
							break;
						}  
					} 	
				}
			}
			xmlr6.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
			log.info("Problem getting ServiceKey" + e.toString());
		}
			//"/Sources/Source/Export/SiteService[Key/text() = \"" + siteServiceID+"\"]/ServiceID/text()", doc, XPathConstants.NODESET); 
		return serviceKey;
	}
	
	ArrayList<SiteServiceParticipation> getSiteServiceParticipationsForHousehold(Household household, BlobKey hudXMLURL) {
		//return value
		ArrayList<SiteServiceParticipation> sspArrayList = new ArrayList<SiteServiceParticipation>();
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);
		XMLStreamReader xmlr7 = null;

		try {
			// set up a StAX reader
			xmlr7 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
		
		//Use StAX to retrieve serviceID (proxy for agencyID) from the SiteService
		try {
			jc = JAXBContext.newInstance(SiteServiceParticipation.class);
		} catch (JAXBException e) {
			log.info("JAXB SiteServiceParticipation.class context problem: " + e.toString());
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling SiteServiceParticipation context: " + e.toString());
		}
		try {
			while(xmlr7.hasNext()) {
				//Get and check each SiteService for the right SiteServiceID
				xmlr7.next();
				if (xmlr7.isStartElement()) {
					//System.out.println("local name is: " + xmlr.getLocalName());
					if (xmlr7.getLocalName().equals("SiteServiceParticipation")) {
						xmlr7.require(XMLStreamConstants.START_ELEMENT, null, "SiteServiceParticipation");
						JAXBElement<SiteServiceParticipation> siteServiceParticipationJAXB = unmarshaller.unmarshal(xmlr7, SiteServiceParticipation.class);
						SiteServiceParticipation siteServiceParticipationElement = (SiteServiceParticipation)siteServiceParticipationJAXB.getValue();
						//Get only this household's SiteServiceParticipations 
						try {
							if (siteServiceParticipationElement.getHouseholdID().getIDStr() != null) {
								if ((siteServiceParticipationElement.getHouseholdID().getIDStr()).equals(household.getHouseholdID().getIDStr())) {
									log.info("siteServiceParticipationElement.getHouseholdID().getIDStr is a match for household: " + household.getHouseholdID().getIDStr());
									sspArrayList.add(siteServiceParticipationElement);
									log.info("Added an ssp with id: " + siteServiceParticipationElement.getSiteServiceParticipationID().getIDNum().toString() + " for household:" + household.getHouseholdID().getIDStr());
								}
							} else if (siteServiceParticipationElement.getHouseholdID().getIDNum() != null) {
								if ((siteServiceParticipationElement.getHouseholdID().getIDNum().toString()).equals(household.getHouseholdID().getIDNum())) {
									log.info("siteServiceParticipationElement.getHouseholdID().getIDNum is a match for household: " + household.getHouseholdID().getIDNum());
									sspArrayList.add(siteServiceParticipationElement);
									log.info("Added an ssp with id: " + siteServiceParticipationElement.getSiteServiceParticipationID().getIDNum().toString() + " for household:" + household.getHouseholdID().getIDNum());
								}
							}
						} catch (NullPointerException e) {
							//log.info("siteServiceParticipationElement.getHouseholdID().getIDStr is null for household: " + household.getHouseholdID().getIDStr());
						}
					} 	
				}
			}
			xmlr7.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
		}
		return sspArrayList;
	}	
	
	String getHeadOfHouseholdSSNFromHousehold(Household household, BlobKey hudXMLURL) {
		if (household.getHouseholdID().getIDStr() != null) {
			log.info("Retrieving head of household's SSN for household ID: " + household.getHouseholdID().getIDStr() + " and head of household id: " + household.getHeadOfHouseholdID().getIDNum().toString());
		} else if (household.getHouseholdID().getIDNum() != null) {
			log.info("Retrieving head of household's SSN for household ID: " + household.getHouseholdID().getIDNum() + " and head of household id: " + household.getHeadOfHouseholdID().getIDNum().toString());

		}
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);
		XMLStreamReader xmlr8 = null;
		try {
			// set up a StAX reader
			xmlr8 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}

		//Use StAX to retrieve the head of household's ssn from the Household
		String ssn = "";
		
		try {
			jc = JAXBContext.newInstance(Person.class);
		} catch (JAXBException e) {
			log.info("JAXB Person.class context problem: " + e.toString());
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Person context: " + e.toString());
		}
		try {
			while(xmlr8.hasNext()) {
				//Get and check each Person with the right Head of Household ID
				xmlr8.next();
				if (xmlr8.isStartElement()) {
					if (xmlr8.getLocalName().equals("Person")) {
						xmlr8.require(XMLStreamConstants.START_ELEMENT, null, "Person");
						JAXBElement<Person> personJAXB = unmarshaller.unmarshal(xmlr8, Person.class);
						Person person = (Person)personJAXB.getValue();
						//Get only the person with the same id as the head of household's id 
						if (person.getPersonID().getIDNum().equals(household.getHeadOfHouseholdID().getIDNum())) {
							ssn = person.getSocialSecurityNumber().getUnhashed().getValue();
							break;
						}  
					} 	
				}
			}
			xmlr8.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
		}
		return ssn;
	}	
	
	Person getPersonFromSiteServiceParticipation(SiteServiceParticipation ssp, BlobKey hudXMLURL) {
		Person personReturn = null;
		JAXBContext jc = null;

		InputStream zis = openXMLStream(hudXMLURL);

		XMLStreamReader xmlr9 = null;
		try {
			// set up a StAX reader
			xmlr9 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
		
		BigInteger sspID = ssp.getSiteServiceParticipationID().getIDNum();
		// Use StAX to get Person that possesses that ssp id
		try {
			jc = JAXBContext.newInstance(Person.class);
		} catch (JAXBException e) {
			log.info("JAXB Person.class context problem: " + e.toString());
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Person context: " + e.toString());
		}
		try {
			while(xmlr9.hasNext()) {
				//Get and check each Person with the right Head of Household ID
				xmlr9.next();
				if (xmlr9.isStartElement()) {
					if (xmlr9.getLocalName().equals("Person")) {
						xmlr9.require(XMLStreamConstants.START_ELEMENT, null, "Person");
						JAXBElement<Person> personJAXB = unmarshaller.unmarshal(xmlr9, Person.class);
						Person person = (Person)personJAXB.getValue();
						//Get only the person with the same id as the head of household's id 
						for (SiteServiceParticipation ssp2 : person.getSiteServiceParticipation()) {
							if (ssp2.getSiteServiceParticipationID().getIDNum().equals(sspID)) {
								personReturn = person;
								break;
							}	
						}
					} 	
				}
			}
			xmlr9.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
		}
		return personReturn;
	}
	
	Person getPersonFromHouseholdMember(Member member, BlobKey hudXMLURL) {
		Person personReturn = null;
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);
		XMLStreamReader xmlr10 = null;
		try {
			// set up a StAX reader
			xmlr10 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
		}
		
		// Use StaX to search for a Person with the same ID as the household member
		try {
			jc = JAXBContext.newInstance(Person.class);
		} catch (JAXBException e) {
			log.info("JAXB Person.class context problem: " + e.toString());
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Person context: " + e.toString());
		}
		try {
			while(xmlr10.hasNext()) {
				//Get and check each Person with the right Head of Household ID
				xmlr10.next();
				if (xmlr10.isStartElement()) {
					if (xmlr10.getLocalName().equals("Person")) {
						xmlr10.require(XMLStreamConstants.START_ELEMENT, null, "Person");
						JAXBElement<Person> personJAXB = unmarshaller.unmarshal(xmlr10, Person.class);
						Person person = (Person)personJAXB.getValue();
						//Get only the person with the same id as the head of household's id 
						if (person.getPersonID().getIDNum().equals(member.getPersonID().getIDNum())) {
								personReturn = person;
								break;
						}
					} 	
				}
			}
			xmlr10.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
		}
		return personReturn;
	}
	
	void convertSiteServiceParticipationtoRosieCSV1(SiteServiceParticipation ssp, BlobKey hudXMLURL) {
		// populate ROSIE CSV file 1 "Household Information" with each household's program enrollment
		// each person's enrollment using the associated household represents one record in Rosie #1
		log.info("inside convertSiteServiceParticipationtoRosieCSV()");
		file1Record = new String[27];
		String sSNFamilyHeadCSV1;
		if (ssp.getHouseholdID().getIDStr() != null) {
			log.info("using household id string version in convertSiteServiceParticipationtoRosieCSV1");
			sSNFamilyHeadCSV1 = getHeadOfHouseholdSSNFromHouseholdID(ssp.getHouseholdID().getIDStr(), hudXMLURL);
		} else if (ssp.getHouseholdID().getIDNum() != null) {
			sSNFamilyHeadCSV1 = getHeadOfHouseholdSSNFromHouseholdID(ssp.getHouseholdID().getIDNum().toString(), hudXMLURL);
			log.info("using household id number version in convertSiteServiceParticipationtoRosieCSV1");
		} else sSNFamilyHeadCSV1 = null;
		
		log.info("Placing \'" + sSNFamilyHeadCSV1 + "\' into file1Record[3]");
		file1Record[3] = sSNFamilyHeadCSV1;
		
		//TransactionDate_mmddyy - use SiteServiceParticipationID's Participation Start Date
		String transactionDateCSV1 = null;
		String transactionTimeCSV1 = null;
		
		GregorianCalendar calendar = ssp.getParticipationDates().getStartDate().toGregorianCalendar();
		SimpleDateFormat formatOutDate = new SimpleDateFormat("MMddyy");
		SimpleDateFormat formatOutTime = new SimpleDateFormat("HHmmss");
		formatOutDate.setCalendar(calendar);
		formatOutTime.setCalendar(calendar);

		//SimpleDateFormat formatIn = new SimpleDateFormat("yyyy-MM-dd'T'hh':'mm':'ss");
		Date date;
		date = calendar.getTime();
		transactionDateCSV1 = formatOutDate.format(date);
		transactionTimeCSV1 = formatOutTime.format(date);
	
		file1Record[1] = transactionDateCSV1;
		file1Record[2] = transactionTimeCSV1;
		
		//Use head of household's id to look up Person record's address info
		BigInteger headOfHouseholdID;
		if (ssp.getHouseholdID().getIDStr() != null) {
			headOfHouseholdID = getHeadOfHouseholdIDFromHouseholdID(ssp.getHouseholdID().getIDStr(), hudXMLURL);
		} else if (ssp.getHouseholdID().getIDNum() != null) {
			headOfHouseholdID = getHeadOfHouseholdIDFromHouseholdID(ssp.getHouseholdID().getIDNum().toString(), hudXMLURL);
		} else headOfHouseholdID = null;
		Person headOfHousehold = getPersonByID(headOfHouseholdID, hudXMLURL);
		Household household = null;
		if (ssp.getHouseholdID().getIDStr() != null) {
			household = getHouseholdFromHouseholdID(ssp.getHouseholdID().getIDStr(), hudXMLURL);
		} else if (ssp.getHouseholdID().getIDNum() != null) {
			household = getHouseholdFromHouseholdID(ssp.getHouseholdID().getIDNum().toString(), hudXMLURL);
		} else {log.info("Couldn't look up household.");}
		
		//For CSV 1 Household Information, we look inside the siteserviceparticipation's personhistorical, but we should look into unnested personhistoricals as well
		log.info("Looking up PersonID: " + headOfHouseholdID.toString() + " Addresses");
		
		//Get the head of household's address
		
			//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + headOfHouseholdID + "\"]/SiteServiceParticipation/PersonHistorical/PersonAddress", doc, XPathConstants.NODESET);
			//"/Sources/Source/Export/Person[PersonID/IDStr/text() = \"" + headOfHouseholdID + "\"]/SiteServiceParticipation/PersonHistorical/PersonAddress", doc, XPathConstants.NODESET);					
		List<SiteServiceParticipation> ssps = headOfHousehold.getSiteServiceParticipation();
		for (SiteServiceParticipation sspHeadOfHousehold : ssps) {
			List<PersonHistorical> phs = sspHeadOfHousehold.getPersonHistorical();
			for (PersonHistorical ph : phs) {
				try {
					PersonAddress personAddress = ph.getPersonAddress().get(0);
					if (personAddress  != null) {
						//for (PersonAddress personAddress : personAddresses) {
						//Process each address found, either temporary or last permanent
						//log.info("We located " + numberOfAddresses + " total addresses for the head of the household.");				
						// first, for each address in the list, check the lastpermanentzip element to make sure which csv column it goes in
						
						Integer isLastPermanentZIP = ((Long)personAddress.getIsLastPermanentZIP().getValue()).intValue();
						boolean isLastPermanent;
						//isLastPermanentZIP = xpath.evaluate("IsLastPermanentZIP", personAddress);
						//log.info("Problem getting IsLastPermanentZIP: " + e.toString());
						if (isLastPermanentZIP.equals("1")) {
								// put fields in Last Permanent Address columns
								isLastPermanent = true;
						} else if (isLastPermanentZIP.equals("0")) {
								// put fields in Temp Address columns
								isLastPermanent = false;
						} else {//log.info("isLastPermanentZIP == neither 1 nor 0");
							isLastPermanent = false;
						}
						// then, for each address for that person, process it
						processPersonAddressHeadOfHousehold(headOfHouseholdID.toString(), isLastPermanent, personAddress);

						// process PersonPhoneNumber element
						//HUD HMIS XML doesn't differentiate between work and home telephone numbers, so we just get the first match and put it in home
						//String homeTelephoneCSV1= ((Element)personPhoneList.item(0)).getTextContent();										file1Record[16] = "";					
						PhoneNumber personPhoneNumber = ph.getPersonPhoneNumber().get(0);
						//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + headOfHouseholdID + "\"]/SiteServiceParticipation/PersonHistorical/PersonPhoneNumber", doc, XPathConstants.NODESET);
						String homeTelephoneCSV1 = "";
						//if (!personPhoneNumbers.isEmpty()) {
						homeTelephoneCSV1 = personPhoneNumber.getValue();
						file1Record[16] = homeTelephoneCSV1;
						//log.info("Problem getting PersonPhoneNumber: " + e.toString());
					
						// calculate FamilySize for this household (HUD HMIS XML doesn't use this; it just lists the members) 
						// This is a very simplistic calculation, and doesn't dedup nor search by all elements with same household id
						//"/Sources/Source/Export/Household[HouseholdID/IDNum/text() = \"" + hid + "\"]/Members/Member", doc, XPathConstants.NODESET);
						//"/Sources/Source/Export/Household[HouseholdID/IDStr/text() = \"" + hid + "\"]/Members/Member", doc, XPathConstants.NODESET);
						try {	
							List<Member> members= household.getMembers().getMember();
							String familySizeCSV1= String.valueOf(members.size());
							file1Record[18] = familySizeCSV1; 
							//log.info("Problem getting household members: " + e.toString());
						} catch (NullPointerException e) {
							if (household.getHouseholdID().getIDStr() != null) {
								log.info("For family size, no members in household: " + household.getHouseholdID().getIDStr());
							} else if (household.getHouseholdID().getIDNum() != null) {
								log.info("For family size, no members in household: " + household.getHouseholdID().getIDNum());
							}
							log.info("So set family size equal to '1' for the head of household indicated");
							file1Record[18] = "1"; 
						}
						
						// get housing situation code; look up by the head of household's id
						List<SixValDKRefused2> housingStatuses = ph.getHousingStatus();
						String housingSituationCodeCSV1 = "";
						if (!housingStatuses.isEmpty()) {
							 housingSituationCodeCSV1 = String.valueOf(housingStatuses.get(0).getValue());							
						}
						//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + headOfHouseholdID + "\"]/SiteServiceParticipation/PersonHistorical/HousingStatus/text()", 
						//if (housingSituationCodeCSV1 == null | housingSituationCodeCSV1.isEmpty())
						file1Record[19] = housingSituationCodeCSV1;
						//log.info("Problem getting housing status: " + e.toString());

						// get length in housing situation code; look up by the head of household's id
						String lengthInHousingSituationCodeCSV1 ="";
						List<SevenValDKRefused2> lengthOfStays = ph.getLengthOfStayAtPriorResidence();
						if (!lengthOfStays.isEmpty()) {
							lengthInHousingSituationCodeCSV1 = String.valueOf(lengthOfStays.get(0).getValue());
						}
							//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + headOfHouseholdID + "\"]/SiteServiceParticipation/PersonHistorical/LengthOfStayAtPriorResidence/text()", 
							//if (lengthInHousingSituationCodeCSV1 == null | lengthInHousingSituationCodeCSV1.isEmpty()) {
						file1Record[20] = lengthInHousingSituationCodeCSV1;
						//log.info("Problem getting Length of Stay: " + e.toString());
						
						// get Prior_Loc; look up by the head of household's id
						List<PriorResidence> priorResidences = ph.getPriorResidence();
						String priorLocCSV1 = "";
						if (!priorResidences.isEmpty()) {
							priorLocCSV1 = String.valueOf(priorResidences.get(0).getPriorResidenceCode().getValue());
						}
						//"/Sources/Source/Export/Person[PersonID/IDNum/text() = \" + headOfHouseholdID + "\"]/SiteServiceParticipation/PersonHistorical/PriorResidence/PriorResidenceCode/text()"
						//if (priorLocCSV1 == null | priorLocCSV1.isEmpty()) {
						file1Record[25] = priorLocCSV1;
						//log.info("Problem getting prior location: " + e.toString());

						//write out the Family Income and Expense record to CSV3
						processIncomeAndSources(household, ph, ssp, hudXMLURL);
						//clear record
						for (int i = 0; i < file3Record.length; i++) 
							file3Record[i] = "";
						
						//write out the Member Health Problems record to CSV4
						processHealthStatus(household, hudXMLURL);
						//clear record
						for (int i = 0; i < file4Record.length; i++) 
							file4Record[i] = "";
						
						//write out the Family Assistance Provided record to CSV6
						processFundingSourceServiceEvent(household, hudXMLURL);

					}
				} catch (Exception e) {
					log.info("Problem getting a personAddress; there may not have been one in the personhistorical.");
				}
			}
		}
	// write out the whole record into CSV 1
		log.info("writing out file 1");
		try {
			FileWriteChannel fwc1 = fileService.openWriteChannel(fileRosieCSV1FamilyInformation, false);
			PrintWriter pw1 = new PrintWriter(Channels.newWriter(fwc1, "UTF8"));
			CSVWriter writer1 = new CSVWriter(new BufferedWriter(pw1));
			writeNextAndClose(writer1, fwc1, pw1, fileRosieCSV1FamilyInformation, file1Record);
		} catch (IOException ioe) {ioe.printStackTrace();}		//clear record
		for (int i = 0; i < file1Record.length; i++) 
			file1Record[i] = "";
	}
	
	String getHeadOfHouseholdSSNFromHouseholdID(String householdID, BlobKey hudXMLURL) {
		JAXBContext jc = null;
		//Use StAX to retrieve the head of household's ssn from the Household ID
		//First, use the Household ID to get the household element.
		InputStream zis = openXMLStream(hudXMLURL);
		String ssn = "";
		XMLStreamReader xmlr13 = null;
		try {
			// set up a StAX reader
			xmlr13 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
			e.printStackTrace();
		}
		try {
			jc = JAXBContext.newInstance(Household.class);
		} catch (JAXBException e) {
			log.info("JAXB Household.class context problem: " + e.toString());
			e.printStackTrace();
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Household.class context: " + e.toString());
			e.printStackTrace();
		}
		try {
			while(xmlr13.hasNext()) {
				//Get and check each Household with the right Household ID
				xmlr13.next();
				if (xmlr13.isStartElement()) {
					if (xmlr13.getLocalName().equals("Household")) {
						xmlr13.require(XMLStreamConstants.START_ELEMENT, null, "Household");
						JAXBElement<Household> householdJAXB = unmarshaller.unmarshal(xmlr13, Household.class);
						Household household = (Household)householdJAXB.getValue();
						if (household.getHouseholdID().getIDStr() != null) {
							if (household.getHouseholdID().getIDStr().equals(householdID)) {
							//Then, use getHeadOfHouseholdSSNFromHousehold() method
								ssn = getHeadOfHouseholdSSNFromHousehold(household, hudXMLURL);
								break;
							}
						} else if (household.getHouseholdID().getIDNum() != null) {
							if ((household.getHouseholdID().getIDNum().toString()).equals(householdID)) {
							//Then, use getHeadOfHouseholdSSNFromHousehold() method
								ssn = getHeadOfHouseholdSSNFromHousehold(household, hudXMLURL);
								break;
							}
						}
					} 	
				}
			}
			xmlr13.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
			e.printStackTrace();
		}
		return ssn;
	}
	
	BigInteger getHeadOfHouseholdIDFromHouseholdID(String householdID, BlobKey hudXMLURL) {
		//Use StAX to retrieve the head of household's id from the Household ID
		//First, use the Household ID to get the household element.
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);
		BigInteger headOfHouseholdID = null;
		XMLStreamReader xmlr14 = null;
		try {
			// set up a StAX reader
			xmlr14 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
			e.printStackTrace();
		}
		try {
			jc = JAXBContext.newInstance(Household.class);
		} catch (JAXBException e) {
			log.info("JAXB Person.class context problem: " + e.toString());
			e.printStackTrace();
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling Person context: " + e.toString());
			e.printStackTrace();
		}
		try {
			while(xmlr14.hasNext()) {
				//Get and check each Household with the right Household ID
				xmlr14.next();
				if (xmlr14.isStartElement()) {
					if (xmlr14.getLocalName().equals("Household")) {
						xmlr14.require(XMLStreamConstants.START_ELEMENT, null, "Household");
						JAXBElement<Household> householdJAXB = unmarshaller.unmarshal(xmlr14, Household.class);
						Household household = (Household)householdJAXB.getValue();
						if (household.getHouseholdID().getIDStr() != null) {
							if (household.getHouseholdID().getIDStr().equals(householdID)) {
								headOfHouseholdID = household.getHeadOfHouseholdID().getIDNum();
								break;
							}
						} else if (household.getHouseholdID().getIDNum() != null) {
							if ((household.getHouseholdID().getIDNum().toString()).equals(householdID)) {
								headOfHouseholdID = household.getHeadOfHouseholdID().getIDNum();
								break;
							}
						}
					} 	
				}
			}
			xmlr14.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
			e.printStackTrace();
		}
		return headOfHouseholdID;
	}
	
	Person getPersonByID(BigInteger headOfHouseholdID, BlobKey hudXMLURL) {
		Person returnPerson = null;
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);
		XMLStreamReader xmlr15 = null;
		try {
			// set up a StAX reader
			xmlr15 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
			e.printStackTrace();
		}
		
		//Use StAX to look up the head of household's SSN
		try {
			jc = JAXBContext.newInstance(Person.class);
		} catch (JAXBException e) {
			log.info("JAXB Person.class context problem: " + e.toString());
			e.printStackTrace();
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info("Problem unmarshalling person context: " + e.toString());
			e.printStackTrace();
		}
		try {
			while(xmlr15.hasNext()) {
				//Get each person
				xmlr15.next();
				if (xmlr15.isStartElement()) {
					if (xmlr15.getLocalName().equals("Person")) {
						xmlr15.require(XMLStreamConstants.START_ELEMENT, null, "Person");
						JAXBElement<Person> personJAXB = unmarshaller.unmarshal(xmlr15, Person.class);
						Person person = (Person)personJAXB.getValue();
						if (person.getPersonID().getIDNum().equals(headOfHouseholdID)) {
							returnPerson = person;
							break;
						}
					}
				}
			}
			xmlr15.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
			e.printStackTrace();
		}
		return returnPerson;
	}
	
	Household  getHouseholdFromHouseholdID(String householdID, BlobKey hudXMLURL) {
		Household returnHousehold = null;
		log.info("Attempting to get household for household id: " + householdID);
		//Using StAX to get all the household ids for File 2 Household Membership Generation
		JAXBContext jc = null;
		InputStream zis = openXMLStream(hudXMLURL);

		XMLStreamReader xmlr15 = null;
		try {
			
			// set up a StAX reader
			xmlr15 = xmlif.createXMLStreamReader(zis);
		} catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
			e.printStackTrace();
		}
		try {
			jc = JAXBContext.newInstance(Household.class);
		} catch (JAXBException e) {
			e.toString();
			e.printStackTrace();
		}
		Unmarshaller unmarshaller = null;
		try {
			unmarshaller = jc.createUnmarshaller();
		} catch (JAXBException e) {
			log.info(e.toString());
			e.printStackTrace();
		}
		try {
			while(xmlr15.hasNext()) {
				//Get each household
				xmlr15.next();
				if (xmlr15.isStartElement()) {
					//log.info("name is:" + xmlr15.getLocalName());
					if (xmlr15.getLocalName().equals("Household")  && xmlr15.isStartElement()) {
						xmlr15.require(XMLStreamConstants.START_ELEMENT, null, "Household");
						JAXBElement<Household> householdJAXB = unmarshaller.unmarshal(xmlr15,Household.class);
						Household householdElement = (Household)householdJAXB.getValue();
						if (householdElement.getHouseholdID().getIDStr() != null) {
							if (householdElement.getHouseholdID().getIDStr().equals(householdID)) {
								log.info("found a matching household for household id: " + householdID);
								returnHousehold = householdElement;
								break;
							}
						} else if (householdElement.getHouseholdID().getIDNum() != null) {
							if (householdElement.getHouseholdID().getIDNum().toString().equals(householdID)) {
								log.info("found a matching household for household id: " + householdID);
								returnHousehold = householdElement;
								break;
							}
						} else {log.info("Couldn't find a household match in getHouseholdFromHouseholdID.");}
					}
				}
			}
			xmlr15.close();
			zis.close();
		} catch (Exception e) {
			log.info("XML Stream exception caught: " + e.toString());
			e.printStackTrace();
		}
		return returnHousehold;
	}
	
	void processMembersAndHeadsOfHouseholds(Household household, Object o, BlobKey hudXMLURL) {
		JAXBContext jc = null;
		log.info("inside processHouseholdMemberAndHeadsOfHouseholds()");
		InputStream zis = openXMLStream(hudXMLURL);

		XMLStreamReader xmlr4 = null;
		try {

			// set up a StAX reader
			xmlr4 = xmlif.createXMLStreamReader(zis);
		}   catch (Exception e) {
			log.info("exception caught creating StAX stream " + e.toString());
			e.printStackTrace();
		}
		if (household.getHouseholdID().getIDStr() !=null) {
			log.info("searching for household id: " + household.getHouseholdID().getIDStr() + " in any site service participation, and person idnum: ");// + member.getPersonID().getIDNum().toString() + " to get a siteServiceID");
		} else if (household.getHouseholdID().getIDNum() !=null) {
			log.info("searching for household id: " + household.getHouseholdID().getIDNum() + " in any site service participation, and person idnum: ");// + member.getPersonID().getIDNum().toString() + " to get a siteServiceID");
		} else {log.info("couldn't extract ID (string or num from the household element");}
		
		if (o instanceof Member) {
			String memberid = ((Member)o).getPersonID().getIDNum().toString();
				log.info(memberid);
		}  else if (o instanceof BigInteger) {
			log.info(((BigInteger)o).toString());
		} else {log.info("can't get idnum of household member");}
		//Get Service Key (Rosie would normally get AgencyCode) from a SiteServiceParticipation the household member person has
		//This is problematic, since this household id might have received services from many agencies.  We should make household records for each of them with a different agency

		//Right now, this only looks at Export/Person/SiteServiceParticipation, but it should also look at Export/SiteServiceParticipation
		//Use StAX to find the matching Person element with the same ID as the household member
		try {
			jc = JAXBContext.newInstance(Person.class);
		}  catch (JAXBException e) {
			log.info("JAXB Person.class context problem: " + e.toString());
			e.printStackTrace();
		}
		try {
			unmarshaller = jc.createUnmarshaller();
		}  catch (JAXBException e) {
			log.info("Problem unmarshalling person context: " + e.toString());
			e.printStackTrace();
		}
		try {
			while(xmlr4.hasNext()) {
				//log.info("While inside xmlr.hasNext()");
				//Get each person
				xmlr4.next();
				if (xmlr4.isStartElement()) {
					if (xmlr4.getLocalName().equals("Person")) {
						//log.info("We found a Person element.");
						xmlr4.require(XMLStreamConstants.START_ELEMENT, null, "Person");
						JAXBElement<Person> personJAXB = unmarshaller.unmarshal(xmlr4, Person.class);
						Person person	= (Person)personJAXB.getValue();		
						if (o instanceof Member) {
							if (((Member)o).getPersonID().getIDNum().equals(person.getPersonID().getIDNum())) {
								log.info("found a match.");
								processMemberStage2(o, household, person, hudXMLURL);
								break;
							} 
						} else if (o instanceof BigInteger) {
							if (((BigInteger)o).equals(person.getPersonID().getIDNum())) {
								//Get the family member's ID
								String familyMemberNumberCSV2 = null;
								try {
									familyMemberNumberCSV2 = ((BigInteger)o).toString();
									//familyMemberNumberCSV2 = xpath.evaluate("PersonID/IDNum/text()", familyMember);
								} catch (Exception e) {
									  log.info(e.toString());
								}
								if (familyMemberNumberCSV2 == null | familyMemberNumberCSV2.isEmpty()) { 
									log.info("No Head of Household PersonID/IDNum found.");
								} else log.info("It apparently found an IDNum (" + familyMemberNumberCSV2 + "), so processing on it should commence");
						
							    log.info("familyMemberNumberCSV2 number we're processing is: " + familyMemberNumberCSV2);
							    file2Record[5] = familyMemberNumberCSV2;
						
								//Get the family member's relationship to the head of household
								String 	relationshipCodeCSV2 = "0";//self
									//relationshipCodeCSV2 = xpath.evaluate("RelationshipToHeadOfHousehold/text()", familyMember);
								file2Record[7] = relationshipCodeCSV2;
								processMemberStage2(o, household, person, hudXMLURL);
								break;
							}
						}
					}
				}
			}
			xmlr4.close();
			zis.close();
		}   catch (Exception e) {
				log.info("XML Stream exception caught: " + e.toString());
				e.printStackTrace();
		}
		//log.info("at end of processMembersAndHeadsOfHouseholds.");
	}

	void processMemberStage2(Object o, Household household, Person person, BlobKey hudXMLURL) {
		String familyMemberSSNCSV2;
		String familyMemberFirstNameCSV2;
		String familyMemberMiddleNameCSV2;
		String familyMemberLastNameCSV2;
		String familyMemberDateOfBirthCSV2;
		String familyMemberGenderCSV2;
		String familyMemberRace1CSV2;
		String familyMemberRace2CSV2;
		String familyMemberHispanicCSV2;
		BigInteger siteServiceID;
		//Get person information
		//TransactionDate_mmddyy - use HeadOfHouseholdID's effectiveDate
		String transactionDateCSV2;
		String transactionTimeCSV2;
		try {
			//log.info("Searching for /Members/Member/PersonID/@dateEffective where hid = "+ hid + "and familyMemberNumberCSV2 is: " + familyMemberNumberCSV2);
			//"/Sources/Source/Export/Household[HouseholdID/IDNum/text() = \"" + hid + "\"]/Members/Member/PersonID[IDNum/text() = \"" + familyMemberNumberCSV2 + "\"]/@dateEffective", doc, XPathConstants.NODESET);
			GregorianCalendar calendar = null;
			if (o instanceof Member ) {
				calendar = ((Member)o).getPersonID().getDateEffective().toGregorianCalendar();
			}
			else if (o instanceof BigInteger) {
				calendar = household.getHeadOfHouseholdID().getDateEffective().toGregorianCalendar();
			}
			SimpleDateFormat formatOutDate = new SimpleDateFormat("MMddyy");
			SimpleDateFormat formatOutTime = new SimpleDateFormat("HHmmss");
			formatOutDate.setCalendar(calendar);
			formatOutTime.setCalendar(calendar);

			//SimpleDateFormat formatIn = new SimpleDateFormat("yyyy-MM-dd'T'hh':'mm':'ss");
			Date date;
			date = calendar.getTime();
			transactionDateCSV2 = formatOutDate.format(date);
			transactionTimeCSV2 = formatOutTime.format(date);
			file2Record[1] = transactionDateCSV2;
			file2Record[2] = transactionTimeCSV2;
		} 
		catch (Exception e) {
			log.info("Couldn't get TransactionDate/Time: " + e.toString());
		}
		try {
			familyMemberSSNCSV2 = person.getSocialSecurityNumber().getUnhashed().getValue();
			file2Record[6] = familyMemberSSNCSV2;
		} 
		catch (Exception e) {
			log.info("Couldn't get SSN: " + e.toString());
		}
		//Get the family member's first name from their Person record
		try {
			familyMemberFirstNameCSV2 = person.getLegalFirstName().getUnhashed().getValue();	
			//log.info("Problem getting LegalFirstName: " + e.toString());
			file2Record[8] = familyMemberFirstNameCSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get First Name: " + e.toString());
		} 
		try {
			//Get the family member's middle name from their Person record
			familyMemberMiddleNameCSV2 = person.getLegalMiddleName().getUnhashed().getValue();	
			//log.info("Problem getting LegalMiddleName: " + e.toString());
			file2Record[9] = familyMemberMiddleNameCSV2;
		}
		catch  (Exception e) {
			log.info("Couldn't get Middle Name: " + e.toString());
		}
		try {
			//Get the family member's last name from their Person record
			familyMemberLastNameCSV2 =  person.getLegalLastName().getUnhashed().getValue();	
			//log.info("Problem getting LegalLastName: " + e.toString());
			file2Record[10] = familyMemberLastNameCSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get Last Name: " + e.toString());
		}
		try {
			//Get the family member's dob from their Person record
			GregorianCalendar calendar2 =  person.getDateOfBirth().getUnhashed().getValue().toGregorianCalendar();	
			//	String xMLDOB =  ((NodeList)xpath.evaluate(
			//		"DateOfBirth/Unhashed/text()", person, XPathConstants.NODESET)).item(0).getTextContent();
			//yyyymmdd->mmddyy
			SimpleDateFormat formatOutDate2 = new SimpleDateFormat("MMddyy");
			formatOutDate2.setCalendar(calendar2);
			Date date2;
			date2 = calendar2.getTime();
			//log.info("Problem parsing DOB: " + e.toString());
			familyMemberDateOfBirthCSV2 = formatOutDate2.format(date2);			
			//log.info("General exception handling DOB: " + e.toString());
			file2Record[11] = familyMemberDateOfBirthCSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get DOB: " + e.toString());
		}
		try {
			//Get the family member's gender from their Person record
			familyMemberGenderCSV2 =  String.valueOf(person.getGender().getUnhashed().getValue());
			//"Gender/Unhashed/text()", person, XPathConstants.NODESET)).item(0).getTextContent();
			//log.info("Problem getting gender: " + e.toString());
			file2Record[12] = familyMemberGenderCSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get Gender: " + e.toString());
		}
		try {						
			//Get the family member's races from their Person record
			//xpath.evaluate("Race", person, XPathConstants.NODESET);
			familyMemberRace1CSV2 = String.valueOf(person.getRace().get(0).getUnhashed().getValue());
			//log.info("Problem getting races: " + e.toString());
			file2Record[13] = familyMemberRace1CSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get Race1: " + e.toString());
		}
		try {						
			familyMemberRace2CSV2 = String.valueOf(person.getRace().get(1).getUnhashed().getValue());				
			file2Record[14] = familyMemberRace2CSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get Race2: " + e.toString());
		}
		try {						
			//Get the family member's ethnicity from their Person record			
			familyMemberHispanicCSV2 = "";
			int ethnicity =  (int) person.getEthnicity().getUnhashed().getValue();
			switch (ethnicity) {
			case 0: familyMemberHispanicCSV2 = "N";
			break;
			case 1: familyMemberHispanicCSV2 = "Y";
			break;
			case 8:;
			break;
			case 9:;
			break;
			default: ;
			}
			//log.info("Caught exception while getting ethnicity: " + e.toString());	
			file2Record[15] = familyMemberHispanicCSV2;
		}
		catch (Exception e) {
			log.info("Couldn't get Ethnicity: " + e.toString());
		}
		//Get all this household member's enrollments
		List <SiteServiceParticipation> ssps = person.getSiteServiceParticipation();
		//Exclude those that don't have the same household id
		for (SiteServiceParticipation ssp : ssps) {
			String hoh_id;
			if (ssp.getHouseholdID().getIDStr() != null) {
				hoh_id = ssp.getHouseholdID().getIDStr();
			} else if (ssp.getHouseholdID().getIDNum() != null) {
				hoh_id = ssp.getHouseholdID().getIDNum().toString();
			} else {hoh_id = "";  log.info("Couldn't get the head of household");}
			
			String hh_id;
			if (household.getHouseholdID().getIDStr() != null) {
				hh_id = ssp.getHouseholdID().getIDStr();
			} else if (household.getHouseholdID().getIDNum() != null) {
				hh_id = household.getHouseholdID().getIDNum().toString();
			} else {hh_id = "";  log.info("Couldn't get the head of household");}
			
			try {
				if (hoh_id.equals(hh_id)) {
					try {
						siteServiceID = ssp.getSiteServiceID();
						log.info("siteServiceID we found in /Sources/Source/Export/Person/SiteServiceParticipation/SiteServiceID for that and ssp of: " + ssp.getSiteServiceParticipationID().getIDNum() + " and  household id of: " + hh_id + " is: " + siteServiceID);
						//list = (NodeList) xpath.evaluate("/Sources/Source/Export/Person[PersonID/IDNum/text() = \"" + familyMemberNumberCSV2 + "\"]/SiteServiceParticipation[HouseholdID/IDNum/text() = \"" + hid+"\"]/SiteServiceID/text()", doc, XPathConstants.NODESET);
						//Now, look up the program participation's parent program/agency/code 
						//First, look up the SiteID within that SiteService with same SiteServiceID (ServiceID wouldn't work in the XML I have)
						//log.info("Got back SiteID of " + siteID + " within same SiteService as siteServiceID " + siteServiceID);
						//log.info("Got back no matching SiteID within same SiteServiceParticipation as siteServiceID" + siteServiceID);
	
						//Now, look up the Service's Key matching the SiteServiceID
						BigInteger serviceKey = null;
						String serviceKeyCSV2 = "";
						//Use Service (Program) as a proxy for agency, if agencies aren't used
						try {
							serviceKey = getServiceIDFromSiteServiceID(siteServiceID, hudXMLURL);
							log.info("Got back Service Key of: " + serviceKey + " from SiteServiceID: " + siteServiceID);
						}
						catch (Exception e) {
							log.info("Got back no Service Key from SiteServiceID: " + siteServiceID);
						}
						//log.info("Got back no Agency Key with same SiteID: " + siteID + ".  Exception: " + e.toString());
						if (!serviceKey.equals(null)) {
							if (serviceKey.signum() == 1) {
								log.info("Got back Service Key: " + serviceKey + " from SiteService ID of: " + siteServiceID);
								serviceKeyCSV2 = serviceKey.toString();
							}  else {
								log.info("Got back no matching ServiceID within same SiteServiceParticipation as siteServiceID: " + siteServiceID);
							}	
						} log.info("Got back no matching ServiceID within same SiteServiceParticipation as siteServiceID: " + siteServiceID);
	
						file2Record[0] = serviceKeyCSV2;
						log.info("file2Record[0] = " + file2Record[0] );
					} catch (Exception e) {
						log.info("Problem getting SiteServiceID: " + e.toString());
						log.info("Got back no matching Service.Key");
					}
	
					//The rest of the CSV2 Family Information records are located in personHistorical, fetch that section for this person
					for (PersonHistorical personHistorical : ssp.getPersonHistorical()) {
						processPersonHistoricalForCSV2(personHistorical);											
					}
				} 
				else {
					log.info("household ids don't match");
				}
			} catch (Exception e) {log.info ("SiteServiceParticipation: " + ssp.getSiteServiceParticipationID().getIDNum().toString() + " doesn't have a householdID?" + e.toString());}
		}
		// write out the whole record into CSV 2
		log.info("Writing out CSV2 Record");
		try {		
			FileWriteChannel fwc2 = fileService.openWriteChannel(fileRosieCSV2FamilyMembers, false);
			PrintWriter pw2 = new PrintWriter(Channels.newWriter(fwc2, "UTF8"));
			CSVWriter writer2 = new CSVWriter(new BufferedWriter(pw2));
			writeNextAndClose(writer2, fwc2, pw2, fileRosieCSV2FamilyMembers, file2Record);
		} catch (IOException ioe) {ioe.printStackTrace();}
		//clear record
		for (int i = 0; i < file2Record.length; i++) {
			file2Record[i] = "";
		}
	}
	void writeNextAndClose(CSVWriter writer, FileWriteChannel fwc,  PrintWriter pw, AppEngineFile appenginefile, String[] record) {
		//actually writes a line to the app engine file, but doesn't finalize it, so not persisted in console, but it's there
		try {
			writer.flush();
			writer.writeNext(record);
			writer.close();
			pw.close();
			fwc.close();
		} catch (IOException e) {
			log.info("Problem closing file in writeNextAndClose: " + e.toString());
		}
	}
}